Build up to date state from pruning proof (#96)

* Add BlockLevel type

* Add block level to relations and ghostdag stores

* calc ghostdag for each level

* Add apply_proof

* fixes to apply_proof

* build state after pruning proof

* fmt

* Fix logger init error

* Make mainnet concurrent test work (+various minor stuff)

* Create DB with increased parallelism

* Fix rebase errors

* Add a Config struct which bundles up all consensus configurations

* sidefix: stop p2p from rebuilding every run

* Use a single lock for all relation levels

* Renames to block task related structs

* Remove async-std crate and use async-channel directly (+use futures-util for stream)

* Add panic configuration to kaspad bin

* Add panic config to simpa and use logger for prints

---------

Co-authored-by: msutton <mikisiton2@gmail.com>

.gitignore

@@ -4,5 +4,6 @@ web-root
 /rust-toolchain
 /.vscode/
 **/db-*
-/consensus/tests/testdata/goref-1.6M-tx-10K-blocks.json.gz
+/consensus/tests/testdata/dags_for_json_tests/goref-mainnet
+/consensus/tests/testdata/dags_for_json_tests/goref-1.6M-tx-10K-blocks
 analyzer-target

Cargo.toml

@@ -62,8 +62,8 @@ criterion = { version = "0.4", default-features = false }
 indexmap = "1.9.1"
 smallvec = { version = "1.10.0", features = ["serde"] }
 borsh = "0.9.3"
-clap = { version = "4.0.23", features = ["derive"] }
-async-std = { version = "1.12.0", features = ['attributes'] }
+clap = { version = "4.0.23", features = ["derive", "string"] }
+async-channel = "1.8.0"
 derive_more = { version = "0.99" }
 log = "0.4"
 cfg-if = "1.0.0"

consensus/core/src/lib.rs

@@ -17,6 +17,7 @@ pub mod merkle;
 pub mod muhash;
 pub mod networktype;
 pub mod notify;
+pub mod pruning;
 pub mod sign;
 pub mod subnets;
 pub mod tx;
@@ -101,6 +102,8 @@ impl BuildHasher for BlockHasher {
     }
 }
 
+pub type BlockLevel = u8;
+
 #[cfg(test)]
 mod tests {
     use super::BlockHasher;

consensus/core/src/muhash.rs

@@ -7,6 +7,7 @@ use muhash::MuHash;
 
 pub trait MuHashExtensions {
     fn add_transaction(&mut self, tx: &impl VerifiableTransaction, block_daa_score: u64);
+    fn add_utxo(&mut self, outpoint: &TransactionOutpoint, entry: &UtxoEntry);
 }
 
 impl MuHashExtensions for MuHash {
@@ -20,11 +21,15 @@ impl MuHashExtensions for MuHash {
         for (i, output) in tx.outputs().iter().enumerate() {
             let outpoint = TransactionOutpoint::new(tx_id, i as u32);
             let entry = UtxoEntry::new(output.value, output.script_public_key.clone(), block_daa_score, tx.is_coinbase());
-            let mut writer = self.add_element_builder();
-            write_utxo(&mut writer, &entry, &outpoint);
-            writer.finalize();
+            self.add_utxo(&outpoint, &entry);
         }
     }
+
+    fn add_utxo(&mut self, outpoint: &TransactionOutpoint, entry: &UtxoEntry) {
+        let mut writer = self.add_element_builder();
+        write_utxo(&mut writer, entry, outpoint);
+        writer.finalize();
+    }
 }
 
 fn write_utxo(writer: &mut impl HasherBase, entry: &UtxoEntry, outpoint: &TransactionOutpoint) {

consensus/core/src/pruning.rs

@@ -0,0 +1,5 @@
+use std::sync::Arc;
+
+use crate::header::Header;
+
+pub type PruningPointProof = Vec<Vec<Arc<Header>>>;

consensus/src/config.rs

@@ -0,0 +1,82 @@
+use std::ops::Deref;
+
+use crate::{
+    constants::perf::{PerfParams, PERF_PARAMS},
+    params::Params,
+};
+
+/// Various consensus configurations all bundled up under a single struct. Use `Config::new` for directly building from
+/// a `Params` instance. For anything more complex it is recommended to use `ConfigBuilder`. NOTE: this struct can be
+/// implicitly de-refed into `Params`
+#[derive(Clone)]
+pub struct Config {
+    /// Consensus params
+    pub params: Params,
+    /// Performance params
+    pub perf: PerfParams,
+
+    //
+    // Additional consensus configuration arguments which are not consensus sensitive
+    //
+    pub process_genesis: bool,
+    // TODO:
+    // is_archival: bool,
+    // enable_sanity_check_pruning_utxoset: bool,
+}
+
+impl Config {
+    pub fn new(params: Params) -> Self {
+        Self { params, perf: PERF_PARAMS, process_genesis: true }
+    }
+}
+
+impl AsRef<Params> for Config {
+    fn as_ref(&self) -> &Params {
+        &self.params
+    }
+}
+
+impl Deref for Config {
+    type Target = Params;
+
+    fn deref(&self) -> &Self::Target {
+        &self.params
+    }
+}
+
+pub struct ConfigBuilder {
+    config: Config,
+}
+
+impl ConfigBuilder {
+    pub fn new(params: Params) -> Self {
+        Self { config: Config::new(params) }
+    }
+
+    pub fn set_perf_params(mut self, perf: PerfParams) -> Self {
+        self.config.perf = perf;
+        self
+    }
+
+    pub fn edit_consensus_params<F>(mut self, edit_func: F) -> Self
+    where
+        F: Fn(&mut Params),
+    {
+        edit_func(&mut self.config.params);
+        self
+    }
+
+    pub fn skip_proof_of_work(mut self) -> Self {
+        self.config.params.skip_proof_of_work = true;
+        self
+    }
+
+    pub fn skip_adding_genesis(mut self) -> Self {
+        self.config.process_genesis = false;
+        self
+    }
+
+    pub fn build(self) -> Config {
+        self.config
+    }
+}