Kaspad with mining support

Cargo.toml

@@ -62,3 +62,4 @@ borsh = "0.9.3"
 clap = { version = "4.0.23", features = ["derive"] }
 async-std = { version = "1.12.0", features = ['attributes'] }
 derive_more = { version = "0.99" }
+log = "0.4"

README.md

@@ -1,22 +1,60 @@
 # Kaspa on Rust
 
-
 Work in progress to implement the Kaspa full-node and related libraries in the Rust programming language.
 
 ## Getting started
 
 - Install the [rust toolchain](https://rustup.rs/).
 
 - Run the following commands:
+
 ```bash
 $ git clone https://github.com/kaspanet/rusty-kaspa
-$ cd rusty-kaspa/kaspad
-$ cargo run --release
+$ cd rusty-kaspa
+```
+
+## Experimenting with the node
+
+The `kaspad` rust executable is currently at the initial stage where a devnet consensus instance can be built and mined locally through the RPC interface. The P2P network is not supported yet. To see it in action, perform the following:
+
+```bash
+$ cargo run --bin kaspad --release
+```
+
+- Download and unzip the latest binaries bundle of [kaspanet/kaspad](https://github.com/kaspanet/kaspad/releases).
+
+- In a separate terminal run the kaspanet/kaspad miner:
+
+```bash
+$ kaspaminer --rpcserver 127.0.0.1:16610 --devnet --miningaddr kaspadev:qrcqat6l9zcjsu7swnaztqzrv0s7hu04skpaezxk43y4etj8ncwfkuhy0zmax
 ```
 
-- This will run a short simulation producing a random DAG and processing it (applying all currently implemented logic).
+- This will create and feed a DAG with the miner getting block templates from the node and submitting them back when mined. The node processes and stores the blocks while applying all currently implemented logic. Execution can be stopped and resumed, the data is persisted in a database.
+
+## Simulation framework (Simpa)
+
+Additionally, the current codebase supports a full in-process network simulation, building an actual DAG over virtual time with virtual delay and benchmarking validation time (following the simulation generation). Execute 
+```bash 
+cargo run --release --bin simpa -- --help
+``` 
+to see the full command line configuration supported by `simpa`. For instance, the following command will run a simulation producing 1000 blocks with communication delay of 2 seconds and BPS=8, and attempts to fill each block with up to 200 transactions.   
+
+```bash
+$ cargo run --release --bin simpa -- -t=200 -d=2 -b=8 -n=1000
+```
+
+## Logging
+
+Logging in `kaspad` and `simpa` can be [filtered](https://docs.rs/env_logger/0.10.0/env_logger/#filtering-results) either by defining the environment variable `RUST_LOG` and/or by adding a `--loglevel` argument to the command, ie.:
+
+```bash
+$ cargo run --bin kaspad -- --loglevel info,rpc_core=trace,rpc_grpc=trace,consensus=trace,kaspa_core=trace
+```
+
+## Tests & Benchmarks
 
 - To run all current tests use:
+
 ```bash
 $ cd rusty-kaspa
 $ cargo test --release
@@ -25,6 +63,7 @@ $ cargo nextest run --release
 ```
 
 - To run current benchmarks:
+
 ```bash
 $ cd rusty-kaspa
 $ cargo bench

consensus/Cargo.toml

@@ -28,6 +28,7 @@ rand.workspace = true
 indexmap.workspace = true
 smallvec.workspace = true
 kaspa-utils.workspace = true
+log.workspace = true
 
 rocksdb = "0.19"
 parking_lot = "0.12"

consensus/core/Cargo.toml

@@ -16,4 +16,5 @@ serde.workspace = true
 faster-hex.workspace = true
 smallvec.workspace = true
 borsh.workspace = true
-secp256k1 = { version = "0.24", features = ["global-context", "rand-std"] }
+secp256k1 = { version = "0.24", features = ["global-context", "rand-std"] }
+futures-util.workspace = true

consensus/core/src/api/mod.rs

@@ -0,0 +1,22 @@
+use futures_util::future::BoxFuture;
+use std::sync::Arc;
+
+use crate::{
+    block::{Block, BlockTemplate},
+    blockstatus::BlockStatus,
+    coinbase::MinerData,
+    tx::Transaction,
+};
+
+/// Abstracts the consensus external API
+pub trait ConsensusApi: Send + Sync {
+    fn build_block_template(self: Arc<Self>, miner_data: MinerData, txs: Vec<Transaction>) -> BlockTemplate;
+
+    fn validate_and_insert_block(
+        self: Arc<Self>,
+        block: Block,
+        update_virtual: bool,
+    ) -> BoxFuture<'static, Result<BlockStatus, String>>;
+}
+
+pub type DynConsensus = Arc<dyn ConsensusApi>;

consensus/core/src/blockstatus.rs

@@ -0,0 +1,31 @@
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Debug)]
+pub enum BlockStatus {
+    /// StatusInvalid indicates that the block is invalid.
+    StatusInvalid,
+
+    /// StatusUTXOValid indicates the block is valid from any UTXO related aspects and has passed all the other validations as well.
+    StatusUTXOValid,
+
+    /// StatusUTXOPendingVerification indicates that the block is pending verification against its past UTXO-Set, either
+    /// because it was not yet verified since the block was never in the selected parent chain, or if the
+    /// block violates finality.
+    StatusUTXOPendingVerification,
+
+    /// StatusDisqualifiedFromChain indicates that the block is not eligible to be a selected parent.
+    StatusDisqualifiedFromChain,
+
+    /// StatusHeaderOnly indicates that the block transactions are not held (pruned or wasn't added yet)
+    StatusHeaderOnly,
+}
+
+impl BlockStatus {
+    pub fn has_block_body(self) -> bool {
+        matches!(self, Self::StatusUTXOValid | Self::StatusUTXOPendingVerification | Self::StatusDisqualifiedFromChain)
+    }
+
+    pub fn is_utxo_valid_or_pending(self) -> bool {
+        matches!(self, Self::StatusUTXOValid | Self::StatusUTXOPendingVerification)
+    }
+}

consensus/core/src/lib.rs

@@ -3,8 +3,10 @@ use std::hash::{BuildHasher, Hasher};
 
 use hashes::Hash;
 
+pub mod api;
 pub mod block;
 pub mod blockhash;
+pub mod blockstatus;
 pub mod coinbase;
 pub mod hashing;
 pub mod header;

consensus/core/src/notify/mod.rs

@@ -3,9 +3,13 @@ use crate::block::Block;
 #[derive(Debug, Clone)]
 pub enum Notification {
     BlockAdded(BlockAddedNotification),
+    NewBlockTemplate(NewBlockTemplateNotification),
 }
 
 #[derive(Debug, Clone)]
 pub struct BlockAddedNotification {
     pub block: Block,
 }
+
+#[derive(Debug, Clone)]
+pub struct NewBlockTemplateNotification {}

consensus/core/src/tx.rs

@@ -1,3 +1,4 @@
+use borsh::{BorshDeserialize, BorshSchema, BorshSerialize};
 use serde::{Deserialize, Serialize};
 use smallvec::SmallVec;
 use std::fmt::Display;
@@ -62,7 +63,8 @@ impl UtxoEntry {
 pub type TransactionIndexType = u32;
 
 /// Represents a Kaspa transaction outpoint
-#[derive(Eq, Hash, PartialEq, Debug, Copy, Clone, Serialize, Deserialize)]
+#[derive(Eq, Hash, PartialEq, Debug, Copy, Clone, Serialize, Deserialize, BorshSerialize, BorshDeserialize, BorshSchema)]
+#[serde(rename_all = "camelCase")]
 pub struct TransactionOutpoint {
     pub transaction_id: TransactionId,
     pub index: TransactionIndexType,

consensus/src/consensus/mod.rs

@@ -21,7 +21,7 @@ use crate::{
             pruning::DbPruningStore,
             reachability::DbReachabilityStore,
             relations::DbRelationsStore,
-            statuses::{BlockStatus, DbStatusesStore, StatusesStoreReader},
+            statuses::{DbStatusesStore, StatusesStoreReader},
             tips::{DbTipsStore, TipsStoreReader},
             utxo_diffs::DbUtxoDiffsStore,
             utxo_multisets::DbUtxoMultisetsStore,
@@ -45,16 +45,19 @@ use crate::{
     },
 };
 use consensus_core::{
+    api::ConsensusApi,
     block::{Block, BlockTemplate},
+    blockstatus::BlockStatus,
     coinbase::MinerData,
     tx::Transaction,
     BlockHashSet,
 };
 use crossbeam_channel::{unbounded, Receiver, Sender};
+use futures_util::future::BoxFuture;
 use hashes::Hash;
 use kaspa_core::{core::Core, service::Service};
 use parking_lot::RwLock;
-use std::future::Future;
+use std::{future::Future, sync::atomic::Ordering};
 use std::{
     ops::DerefMut,
     sync::Arc,
@@ -392,10 +395,11 @@ impl Consensus {
     pub fn validate_and_insert_block(&self, block: Block) -> impl Future<Output = BlockProcessResult<BlockStatus>> {
         let (tx, rx): (BlockResultSender, _) = oneshot::channel();
         self.block_sender.send(BlockTask::Process(block, vec![tx])).unwrap();
+        self.counters.blocks_submitted.fetch_add(1, Ordering::SeqCst);
         async { rx.await.unwrap() }
     }
 
-    pub fn build_block_template(self: &Arc<Self>, miner_data: MinerData, txs: Vec<Transaction>) -> BlockTemplate {
+    pub fn build_block_template(&self, miner_data: MinerData, txs: Vec<Transaction>) -> BlockTemplate {
         self.virtual_processor.build_block_template(miner_data, txs)
     }
 
@@ -407,6 +411,10 @@ impl Consensus {
         self.statuses_store.read().get(hash).unwrap()
     }
 
+    pub fn processing_counters(&self) -> &Arc<ProcessingCounters> {
+        &self.counters
+    }
+
     pub fn signal_exit(&self) {
         self.block_sender.send(BlockTask::Exit).unwrap();
     }
@@ -420,6 +428,21 @@ impl Consensus {
     }
 }
 
+impl ConsensusApi for Consensus {
+    fn build_block_template(self: Arc<Self>, miner_data: MinerData, txs: Vec<Transaction>) -> BlockTemplate {
+        self.as_ref().build_block_template(miner_data, txs)
+    }
+
+    fn validate_and_insert_block(
+        self: Arc<Self>,
+        block: Block,
+        _update_virtual: bool,
+    ) -> BoxFuture<'static, Result<BlockStatus, String>> {
+        let result = self.as_ref().validate_and_insert_block(block);
+        Box::pin(async move { result.await.map_err(|err| err.to_string()) })
+    }
+}
+
 impl Service for Consensus {
     fn ident(self: Arc<Consensus>) -> &'static str {
         "consensus"

consensus/src/consensus/test_consensus.rs

@@ -5,13 +5,17 @@ use std::{
 };
 
 use consensus_core::{
-    block::{Block, MutableBlock},
+    api::ConsensusApi,
+    block::{Block, BlockTemplate, MutableBlock},
+    blockstatus::BlockStatus,
+    coinbase::MinerData,
     header::Header,
     merkle::calc_hash_merkle_root,
     subnets::SUBNETWORK_ID_COINBASE,
     tx::Transaction,
     BlockHashSet,
 };
+use futures_util::future::BoxFuture;
 use hashes::Hash;
 use kaspa_core::{core::Core, service::Service};
 use parking_lot::RwLock;
@@ -26,7 +30,6 @@ use crate::{
         headers::{DbHeadersStore, HeaderStoreReader},
         pruning::PruningStoreReader,
         reachability::DbReachabilityStore,
-        statuses::BlockStatus,
         DB,
     },
     params::Params,
@@ -38,19 +41,19 @@ use crate::{
 use super::{Consensus, DbGhostdagManager};
 
 pub struct TestConsensus {
-    consensus: Consensus,
+    consensus: Arc<Consensus>,
     pub params: Params,
     temp_db_lifetime: TempDbLifetime,
 }
 
 impl TestConsensus {
     pub fn new(db: Arc<DB>, params: &Params) -> Self {
-        Self { consensus: Consensus::new(db, params), params: params.clone(), temp_db_lifetime: Default::default() }
+        Self { consensus: Arc::new(Consensus::new(db, params)), params: params.clone(), temp_db_lifetime: Default::default() }
     }
 
     pub fn create_from_temp_db(params: &Params) -> Self {
         let (temp_db_lifetime, db) = create_temp_db();
-        Self { consensus: Consensus::new(db, params), params: params.clone(), temp_db_lifetime }
+        Self { consensus: Arc::new(Consensus::new(db, params)), params: params.clone(), temp_db_lifetime }
     }
 
     pub fn build_header_with_parents(&self, hash: Hash, parents: Vec<Hash>) -> Header {
@@ -102,7 +105,7 @@ impl TestConsensus {
     }
 
     pub fn validate_and_insert_block(&self, block: Block) -> impl Future<Output = BlockProcessResult<BlockStatus>> {
-        self.consensus.validate_and_insert_block(block)
+        self.consensus.as_ref().validate_and_insert_block(block)
     }
 
     pub fn init(&self) -> Vec<JoinHandle<()>> {
@@ -154,6 +157,20 @@ impl TestConsensus {
     }
 }
 
+impl ConsensusApi for TestConsensus {
+    fn build_block_template(self: Arc<Self>, miner_data: MinerData, txs: Vec<Transaction>) -> BlockTemplate {
+        self.consensus.clone().build_block_template(miner_data, txs)
+    }
+
+    fn validate_and_insert_block(
+        self: Arc<Self>,
+        block: Block,
+        update_virtual: bool,
+    ) -> BoxFuture<'static, Result<BlockStatus, String>> {
+        self.consensus.clone().validate_and_insert_block(block, update_virtual)
+    }
+}
+
 impl Service for TestConsensus {
     fn ident(self: Arc<TestConsensus>) -> &'static str {
         "test-consensus"

consensus/src/model/services/statuses.rs

@@ -1,4 +1,5 @@
-use crate::model::stores::statuses::{BlockStatus, StatusesStoreReader};
+use crate::model::stores::statuses::StatusesStoreReader;
+use consensus_core::blockstatus::BlockStatus;
 use hashes::Hash;
 use parking_lot::RwLock;
 use std::sync::Arc;

consensus/src/model/stores/statuses.rs

@@ -1,7 +1,6 @@
-use consensus_core::BlockHasher;
+use consensus_core::{blockstatus::BlockStatus, BlockHasher};
 use parking_lot::{RwLock, RwLockWriteGuard};
 use rocksdb::WriteBatch;
-use serde::{Deserialize, Serialize};
 use std::sync::Arc;
 
 use super::{
@@ -11,36 +10,6 @@ use super::{
 };
 use hashes::Hash;
 
-#[derive(Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Debug)]
-pub enum BlockStatus {
-    /// StatusInvalid indicates that the block is invalid.
-    StatusInvalid,
-
-    /// StatusUTXOValid indicates the block is valid from any UTXO related aspects and has passed all the other validations as well.
-    StatusUTXOValid,
-
-    /// StatusUTXOPendingVerification indicates that the block is pending verification against its past UTXO-Set, either
-    /// because it was not yet verified since the block was never in the selected parent chain, or if the
-    /// block violates finality.
-    StatusUTXOPendingVerification,
-
-    /// StatusDisqualifiedFromChain indicates that the block is not eligible to be a selected parent.
-    StatusDisqualifiedFromChain,
-
-    /// StatusHeaderOnly indicates that the block transactions are not held (pruned or wasn't added yet)
-    StatusHeaderOnly,
-}
-
-impl BlockStatus {
-    pub fn has_block_body(self) -> bool {
-        matches!(self, Self::StatusUTXOValid | Self::StatusUTXOPendingVerification | Self::StatusDisqualifiedFromChain)
-    }
-
-    pub fn is_utxo_valid_or_pending(self) -> bool {
-        matches!(self, Self::StatusUTXOValid | Self::StatusUTXOPendingVerification)
-    }
-}
-
 /// Reader API for `StatusesStore`.
 pub trait StatusesStoreReader {
     fn get(&self, hash: Hash) -> StoreResult<BlockStatus>;