From b14537fac0c69505e64743dba8270c06fa1f11f5 Mon Sep 17 00:00:00 2001 From: D-Stacks <78099568+D-Stacks@users.noreply.github.com> Date: Thu, 19 Sep 2024 22:16:54 +0200 Subject: [PATCH] `virtual chain from block` batching. (#454) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * expose vspc_from_block batching possibilities to rpc. * fmt * limit by merged blocks, set source as def. start. * small clean-up * fmt * actually bound by num of merged blocks, in include transactions case. * fmt * update. * update_2 * new_high = high * remove high hash in consensus api.. as it is not required. * fmt * make proto comment more accurate. * fix tests, and lints, add to ser / der correctly. * change two freq warns to debug * remove option, default to pp. not source. * fix integration test, some Option left. * bump version: ยด0.15.1 => 0.15.2` * remove "optional" startHash * add to cli rpc.rs * remove comment. * edit comment in .proto referencing def. startHash behavior. * only batch added chain blocks, not removed, add check if source is a chain ancestor of high. * remove dangling code in comment * remove error from some prev. commit. * Optionalize limts. --------- Co-authored-by: Michael Sutton --- Cargo.lock | 116 +++++++++--------- Cargo.toml | 112 ++++++++--------- cli/src/modules/rpc.rs | 18 ++- components/consensusmanager/src/session.rs | 16 ++- consensus/core/src/api/mod.rs | 13 +- consensus/src/consensus/mod.rs | 50 ++++++-- .../pipeline/virtual_processor/processor.rs | 2 +- consensus/src/processes/sync/mod.rs | 2 +- consensus/src/processes/traversal_manager.rs | 19 ++- mining/src/mempool/remove_transaction.rs | 6 +- rpc/grpc/core/proto/rpc.proto | 9 +- rpc/service/src/converter/consensus.rs | 3 +- rpc/service/src/service.rs | 21 +++- .../src/daemon_integration_tests.rs | 8 +- 14 files changed, 247 insertions(+), 148 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7f23aca0f..1296a8922 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2363,7 +2363,7 @@ dependencies = [ [[package]] name = "kaspa-addresses" -version = "0.15.1" +version = "0.15.2" dependencies = [ "borsh", "criterion", @@ -2380,7 +2380,7 @@ dependencies = [ [[package]] name = "kaspa-addressmanager" -version = "0.15.1" +version = "0.15.2" dependencies = [ "borsh", "igd-next", @@ -2403,14 +2403,14 @@ dependencies = [ [[package]] name = "kaspa-alloc" -version = "0.15.1" +version = "0.15.2" dependencies = [ "mimalloc", ] [[package]] name = "kaspa-bip32" -version = "0.15.1" +version = "0.15.2" dependencies = [ "borsh", "bs58", @@ -2437,7 +2437,7 @@ dependencies = [ [[package]] name = "kaspa-cli" -version = "0.15.1" +version = "0.15.2" dependencies = [ "async-trait", "borsh", @@ -2484,7 +2484,7 @@ dependencies = [ [[package]] name = "kaspa-connectionmanager" -version = "0.15.1" +version = "0.15.2" dependencies = [ "duration-string", "futures-util", @@ -2501,7 +2501,7 @@ dependencies = [ [[package]] name = "kaspa-consensus" -version = "0.15.1" +version = "0.15.2" dependencies = [ "arc-swap", "async-channel 2.3.1", @@ -2544,7 +2544,7 @@ dependencies = [ [[package]] name = "kaspa-consensus-client" -version = "0.15.1" +version = "0.15.2" dependencies = [ "ahash", "cfg-if 1.0.0", @@ -2572,7 +2572,7 @@ dependencies = [ [[package]] name = "kaspa-consensus-core" -version = "0.15.1" +version = "0.15.2" dependencies = [ "async-trait", "bincode", @@ -2610,7 +2610,7 @@ dependencies = [ [[package]] name = "kaspa-consensus-notify" -version = "0.15.1" +version = "0.15.2" dependencies = [ "async-channel 2.3.1", "cfg-if 1.0.0", @@ -2629,7 +2629,7 @@ dependencies = [ [[package]] name = "kaspa-consensus-wasm" -version = "0.15.1" +version = "0.15.2" dependencies = [ "cfg-if 1.0.0", "faster-hex", @@ -2653,7 +2653,7 @@ dependencies = [ [[package]] name = "kaspa-consensusmanager" -version = "0.15.1" +version = "0.15.2" dependencies = [ "duration-string", "futures", @@ -2671,7 +2671,7 @@ dependencies = [ [[package]] name = "kaspa-core" -version = "0.15.1" +version = "0.15.2" dependencies = [ "cfg-if 1.0.0", "ctrlc", @@ -2689,7 +2689,7 @@ dependencies = [ [[package]] name = "kaspa-daemon" -version = "0.15.1" +version = "0.15.2" dependencies = [ "async-trait", "borsh", @@ -2711,7 +2711,7 @@ dependencies = [ [[package]] name = "kaspa-database" -version = "0.15.1" +version = "0.15.2" dependencies = [ "bincode", "enum-primitive-derive", @@ -2733,7 +2733,7 @@ dependencies = [ [[package]] name = "kaspa-grpc-client" -version = "0.15.1" +version = "0.15.2" dependencies = [ "async-channel 2.3.1", "async-stream", @@ -2765,7 +2765,7 @@ dependencies = [ [[package]] name = "kaspa-grpc-core" -version = "0.15.1" +version = "0.15.2" dependencies = [ "async-channel 2.3.1", "async-stream", @@ -2794,7 +2794,7 @@ dependencies = [ [[package]] name = "kaspa-grpc-server" -version = "0.15.1" +version = "0.15.2" dependencies = [ "async-channel 2.3.1", "async-stream", @@ -2830,7 +2830,7 @@ dependencies = [ [[package]] name = "kaspa-hashes" -version = "0.15.1" +version = "0.15.2" dependencies = [ "blake2b_simd", "borsh", @@ -2851,7 +2851,7 @@ dependencies = [ [[package]] name = "kaspa-index-core" -version = "0.15.1" +version = "0.15.2" dependencies = [ "async-channel 2.3.1", "async-trait", @@ -2870,7 +2870,7 @@ dependencies = [ [[package]] name = "kaspa-index-processor" -version = "0.15.1" +version = "0.15.2" dependencies = [ "async-channel 2.3.1", "async-trait", @@ -2898,7 +2898,7 @@ dependencies = [ [[package]] name = "kaspa-math" -version = "0.15.1" +version = "0.15.2" dependencies = [ "borsh", "criterion", @@ -2919,14 +2919,14 @@ dependencies = [ [[package]] name = "kaspa-merkle" -version = "0.15.1" +version = "0.15.2" dependencies = [ "kaspa-hashes", ] [[package]] name = "kaspa-metrics-core" -version = "0.15.1" +version = "0.15.2" dependencies = [ "async-trait", "borsh", @@ -2942,7 +2942,7 @@ dependencies = [ [[package]] name = "kaspa-mining" -version = "0.15.1" +version = "0.15.2" dependencies = [ "criterion", "futures-util", @@ -2969,7 +2969,7 @@ dependencies = [ [[package]] name = "kaspa-mining-errors" -version = "0.15.1" +version = "0.15.2" dependencies = [ "kaspa-consensus-core", "thiserror", @@ -2977,7 +2977,7 @@ dependencies = [ [[package]] name = "kaspa-muhash" -version = "0.15.1" +version = "0.15.2" dependencies = [ "criterion", "kaspa-hashes", @@ -2990,7 +2990,7 @@ dependencies = [ [[package]] name = "kaspa-notify" -version = "0.15.1" +version = "0.15.2" dependencies = [ "async-channel 2.3.1", "async-trait", @@ -3026,7 +3026,7 @@ dependencies = [ [[package]] name = "kaspa-p2p-flows" -version = "0.15.1" +version = "0.15.2" dependencies = [ "async-trait", "chrono", @@ -3057,7 +3057,7 @@ dependencies = [ [[package]] name = "kaspa-p2p-lib" -version = "0.15.1" +version = "0.15.2" dependencies = [ "borsh", "ctrlc", @@ -3088,7 +3088,7 @@ dependencies = [ [[package]] name = "kaspa-perf-monitor" -version = "0.15.1" +version = "0.15.2" dependencies = [ "kaspa-core", "log", @@ -3100,7 +3100,7 @@ dependencies = [ [[package]] name = "kaspa-pow" -version = "0.15.1" +version = "0.15.2" dependencies = [ "criterion", "js-sys", @@ -3116,7 +3116,7 @@ dependencies = [ [[package]] name = "kaspa-rpc-core" -version = "0.15.1" +version = "0.15.2" dependencies = [ "async-channel 2.3.1", "async-trait", @@ -3158,7 +3158,7 @@ dependencies = [ [[package]] name = "kaspa-rpc-macros" -version = "0.15.1" +version = "0.15.2" dependencies = [ "convert_case 0.6.0", "proc-macro-error", @@ -3170,7 +3170,7 @@ dependencies = [ [[package]] name = "kaspa-rpc-service" -version = "0.15.1" +version = "0.15.2" dependencies = [ "async-trait", "kaspa-addresses", @@ -3199,7 +3199,7 @@ dependencies = [ [[package]] name = "kaspa-testing-integration" -version = "0.15.1" +version = "0.15.2" dependencies = [ "async-channel 2.3.1", "async-trait", @@ -3259,7 +3259,7 @@ dependencies = [ [[package]] name = "kaspa-txscript" -version = "0.15.1" +version = "0.15.2" dependencies = [ "blake2b_simd", "borsh", @@ -3291,7 +3291,7 @@ dependencies = [ [[package]] name = "kaspa-txscript-errors" -version = "0.15.1" +version = "0.15.2" dependencies = [ "secp256k1", "thiserror", @@ -3299,7 +3299,7 @@ dependencies = [ [[package]] name = "kaspa-utils" -version = "0.15.1" +version = "0.15.2" dependencies = [ "arc-swap", "async-channel 2.3.1", @@ -3335,7 +3335,7 @@ dependencies = [ [[package]] name = "kaspa-utils-tower" -version = "0.15.1" +version = "0.15.2" dependencies = [ "cfg-if 1.0.0", "futures", @@ -3349,7 +3349,7 @@ dependencies = [ [[package]] name = "kaspa-utxoindex" -version = "0.15.1" +version = "0.15.2" dependencies = [ "futures", "kaspa-consensus", @@ -3370,7 +3370,7 @@ dependencies = [ [[package]] name = "kaspa-wallet" -version = "0.15.1" +version = "0.15.2" dependencies = [ "async-std", "async-trait", @@ -3382,7 +3382,7 @@ dependencies = [ [[package]] name = "kaspa-wallet-cli-wasm" -version = "0.15.1" +version = "0.15.2" dependencies = [ "async-trait", "js-sys", @@ -3396,7 +3396,7 @@ dependencies = [ [[package]] name = "kaspa-wallet-core" -version = "0.15.1" +version = "0.15.2" dependencies = [ "aes", "ahash", @@ -3477,7 +3477,7 @@ dependencies = [ [[package]] name = "kaspa-wallet-keys" -version = "0.15.1" +version = "0.15.2" dependencies = [ "async-trait", "borsh", @@ -3510,7 +3510,7 @@ dependencies = [ [[package]] name = "kaspa-wallet-macros" -version = "0.15.1" +version = "0.15.2" dependencies = [ "convert_case 0.5.0", "proc-macro-error", @@ -3523,7 +3523,7 @@ dependencies = [ [[package]] name = "kaspa-wallet-pskt" -version = "0.15.1" +version = "0.15.2" dependencies = [ "bincode", "derive_builder", @@ -3550,7 +3550,7 @@ dependencies = [ [[package]] name = "kaspa-wasm" -version = "0.15.1" +version = "0.15.2" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -3578,7 +3578,7 @@ dependencies = [ [[package]] name = "kaspa-wasm-core" -version = "0.15.1" +version = "0.15.2" dependencies = [ "faster-hex", "hexplay", @@ -3589,7 +3589,7 @@ dependencies = [ [[package]] name = "kaspa-wrpc-client" -version = "0.15.1" +version = "0.15.2" dependencies = [ "async-std", "async-trait", @@ -3625,7 +3625,7 @@ dependencies = [ [[package]] name = "kaspa-wrpc-example-subscriber" -version = "0.15.1" +version = "0.15.2" dependencies = [ "ctrlc", "futures", @@ -3640,7 +3640,7 @@ dependencies = [ [[package]] name = "kaspa-wrpc-proxy" -version = "0.15.1" +version = "0.15.2" dependencies = [ "async-trait", "clap 4.5.16", @@ -3659,7 +3659,7 @@ dependencies = [ [[package]] name = "kaspa-wrpc-server" -version = "0.15.1" +version = "0.15.2" dependencies = [ "async-trait", "borsh", @@ -3687,7 +3687,7 @@ dependencies = [ [[package]] name = "kaspa-wrpc-simple-client-example" -version = "0.15.1" +version = "0.15.2" dependencies = [ "futures", "kaspa-rpc-core", @@ -3697,7 +3697,7 @@ dependencies = [ [[package]] name = "kaspa-wrpc-wasm" -version = "0.15.1" +version = "0.15.2" dependencies = [ "ahash", "async-std", @@ -3727,7 +3727,7 @@ dependencies = [ [[package]] name = "kaspad" -version = "0.15.1" +version = "0.15.2" dependencies = [ "async-channel 2.3.1", "cfg-if 1.0.0", @@ -5151,7 +5151,7 @@ dependencies = [ [[package]] name = "rothschild" -version = "0.15.1" +version = "0.15.2" dependencies = [ "async-channel 2.3.1", "clap 4.5.16", @@ -5603,7 +5603,7 @@ dependencies = [ [[package]] name = "simpa" -version = "0.15.1" +version = "0.15.2" dependencies = [ "async-channel 2.3.1", "cfg-if 1.0.0", diff --git a/Cargo.toml b/Cargo.toml index eaf07936e..37acfb172 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -63,7 +63,7 @@ members = [ [workspace.package] rust-version = "1.81.0" -version = "0.15.1" +version = "0.15.2" authors = ["Kaspa developers"] license = "ISC" repository = "https://github.com/kaspanet/rusty-kaspa" @@ -80,61 +80,61 @@ include = [ ] [workspace.dependencies] -# kaspa-testing-integration = { version = "0.15.1", path = "testing/integration" } -kaspa-addresses = { version = "0.15.1", path = "crypto/addresses" } -kaspa-addressmanager = { version = "0.15.1", path = "components/addressmanager" } -kaspa-bip32 = { version = "0.15.1", path = "wallet/bip32" } -kaspa-cli = { version = "0.15.1", path = "cli" } -kaspa-connectionmanager = { version = "0.15.1", path = "components/connectionmanager" } -kaspa-consensus = { version = "0.15.1", path = "consensus" } -kaspa-consensus-core = { version = "0.15.1", path = "consensus/core" } -kaspa-consensus-client = { version = "0.15.1", path = "consensus/client" } -kaspa-consensus-notify = { version = "0.15.1", path = "consensus/notify" } -kaspa-consensus-wasm = { version = "0.15.1", path = "consensus/wasm" } -kaspa-consensusmanager = { version = "0.15.1", path = "components/consensusmanager" } -kaspa-core = { version = "0.15.1", path = "core" } -kaspa-daemon = { version = "0.15.1", path = "daemon" } -kaspa-database = { version = "0.15.1", path = "database" } -kaspa-grpc-client = { version = "0.15.1", path = "rpc/grpc/client" } -kaspa-grpc-core = { version = "0.15.1", path = "rpc/grpc/core" } -kaspa-grpc-server = { version = "0.15.1", path = "rpc/grpc/server" } -kaspa-hashes = { version = "0.15.1", path = "crypto/hashes" } -kaspa-index-core = { version = "0.15.1", path = "indexes/core" } -kaspa-index-processor = { version = "0.15.1", path = "indexes/processor" } -kaspa-math = { version = "0.15.1", path = "math" } -kaspa-merkle = { version = "0.15.1", path = "crypto/merkle" } -kaspa-metrics-core = { version = "0.15.1", path = "metrics/core" } -kaspa-mining = { version = "0.15.1", path = "mining" } -kaspa-mining-errors = { version = "0.15.1", path = "mining/errors" } -kaspa-muhash = { version = "0.15.1", path = "crypto/muhash" } -kaspa-notify = { version = "0.15.1", path = "notify" } -kaspa-p2p-flows = { version = "0.15.1", path = "protocol/flows" } -kaspa-p2p-lib = { version = "0.15.1", path = "protocol/p2p" } -kaspa-perf-monitor = { version = "0.15.1", path = "metrics/perf_monitor" } -kaspa-pow = { version = "0.15.1", path = "consensus/pow" } -kaspa-rpc-core = { version = "0.15.1", path = "rpc/core" } -kaspa-rpc-macros = { version = "0.15.1", path = "rpc/macros" } -kaspa-rpc-service = { version = "0.15.1", path = "rpc/service" } -kaspa-txscript = { version = "0.15.1", path = "crypto/txscript" } -kaspa-txscript-errors = { version = "0.15.1", path = "crypto/txscript/errors" } -kaspa-utils = { version = "0.15.1", path = "utils" } -kaspa-utils-tower = { version = "0.15.1", path = "utils/tower" } -kaspa-utxoindex = { version = "0.15.1", path = "indexes/utxoindex" } -kaspa-wallet = { version = "0.15.1", path = "wallet/native" } -kaspa-wallet-cli-wasm = { version = "0.15.1", path = "wallet/wasm" } -kaspa-wallet-keys = { version = "0.15.1", path = "wallet/keys" } -kaspa-wallet-pskt = { version = "0.15.1", path = "wallet/pskt" } -kaspa-wallet-core = { version = "0.15.1", path = "wallet/core" } -kaspa-wallet-macros = { version = "0.15.1", path = "wallet/macros" } -kaspa-wasm = { version = "0.15.1", path = "wasm" } -kaspa-wasm-core = { version = "0.15.1", path = "wasm/core" } -kaspa-wrpc-client = { version = "0.15.1", path = "rpc/wrpc/client" } -kaspa-wrpc-proxy = { version = "0.15.1", path = "rpc/wrpc/proxy" } -kaspa-wrpc-server = { version = "0.15.1", path = "rpc/wrpc/server" } -kaspa-wrpc-wasm = { version = "0.15.1", path = "rpc/wrpc/wasm" } -kaspa-wrpc-example-subscriber = { version = "0.15.1", path = "rpc/wrpc/examples/subscriber" } -kaspad = { version = "0.15.1", path = "kaspad" } -kaspa-alloc = { version = "0.15.1", path = "utils/alloc" } +# kaspa-testing-integration = { version = "0.15.2", path = "testing/integration" } +kaspa-addresses = { version = "0.15.2", path = "crypto/addresses" } +kaspa-addressmanager = { version = "0.15.2", path = "components/addressmanager" } +kaspa-bip32 = { version = "0.15.2", path = "wallet/bip32" } +kaspa-cli = { version = "0.15.2", path = "cli" } +kaspa-connectionmanager = { version = "0.15.2", path = "components/connectionmanager" } +kaspa-consensus = { version = "0.15.2", path = "consensus" } +kaspa-consensus-core = { version = "0.15.2", path = "consensus/core" } +kaspa-consensus-client = { version = "0.15.2", path = "consensus/client" } +kaspa-consensus-notify = { version = "0.15.2", path = "consensus/notify" } +kaspa-consensus-wasm = { version = "0.15.2", path = "consensus/wasm" } +kaspa-consensusmanager = { version = "0.15.2", path = "components/consensusmanager" } +kaspa-core = { version = "0.15.2", path = "core" } +kaspa-daemon = { version = "0.15.2", path = "daemon" } +kaspa-database = { version = "0.15.2", path = "database" } +kaspa-grpc-client = { version = "0.15.2", path = "rpc/grpc/client" } +kaspa-grpc-core = { version = "0.15.2", path = "rpc/grpc/core" } +kaspa-grpc-server = { version = "0.15.2", path = "rpc/grpc/server" } +kaspa-hashes = { version = "0.15.2", path = "crypto/hashes" } +kaspa-index-core = { version = "0.15.2", path = "indexes/core" } +kaspa-index-processor = { version = "0.15.2", path = "indexes/processor" } +kaspa-math = { version = "0.15.2", path = "math" } +kaspa-merkle = { version = "0.15.2", path = "crypto/merkle" } +kaspa-metrics-core = { version = "0.15.2", path = "metrics/core" } +kaspa-mining = { version = "0.15.2", path = "mining" } +kaspa-mining-errors = { version = "0.15.2", path = "mining/errors" } +kaspa-muhash = { version = "0.15.2", path = "crypto/muhash" } +kaspa-notify = { version = "0.15.2", path = "notify" } +kaspa-p2p-flows = { version = "0.15.2", path = "protocol/flows" } +kaspa-p2p-lib = { version = "0.15.2", path = "protocol/p2p" } +kaspa-perf-monitor = { version = "0.15.2", path = "metrics/perf_monitor" } +kaspa-pow = { version = "0.15.2", path = "consensus/pow" } +kaspa-rpc-core = { version = "0.15.2", path = "rpc/core" } +kaspa-rpc-macros = { version = "0.15.2", path = "rpc/macros" } +kaspa-rpc-service = { version = "0.15.2", path = "rpc/service" } +kaspa-txscript = { version = "0.15.2", path = "crypto/txscript" } +kaspa-txscript-errors = { version = "0.15.2", path = "crypto/txscript/errors" } +kaspa-utils = { version = "0.15.2", path = "utils" } +kaspa-utils-tower = { version = "0.15.2", path = "utils/tower" } +kaspa-utxoindex = { version = "0.15.2", path = "indexes/utxoindex" } +kaspa-wallet = { version = "0.15.2", path = "wallet/native" } +kaspa-wallet-cli-wasm = { version = "0.15.2", path = "wallet/wasm" } +kaspa-wallet-keys = { version = "0.15.2", path = "wallet/keys" } +kaspa-wallet-pskt = { version = "0.15.2", path = "wallet/pskt" } +kaspa-wallet-core = { version = "0.15.2", path = "wallet/core" } +kaspa-wallet-macros = { version = "0.15.2", path = "wallet/macros" } +kaspa-wasm = { version = "0.15.2", path = "wasm" } +kaspa-wasm-core = { version = "0.15.2", path = "wasm/core" } +kaspa-wrpc-client = { version = "0.15.2", path = "rpc/wrpc/client" } +kaspa-wrpc-proxy = { version = "0.15.2", path = "rpc/wrpc/proxy" } +kaspa-wrpc-server = { version = "0.15.2", path = "rpc/wrpc/server" } +kaspa-wrpc-wasm = { version = "0.15.2", path = "rpc/wrpc/wasm" } +kaspa-wrpc-example-subscriber = { version = "0.15.2", path = "rpc/wrpc/examples/subscriber" } +kaspad = { version = "0.15.2", path = "kaspad" } +kaspa-alloc = { version = "0.15.2", path = "utils/alloc" } # external aes = "0.8.3" diff --git a/cli/src/modules/rpc.rs b/cli/src/modules/rpc.rs index f32523c4a..f96cb5b61 100644 --- a/cli/src/modules/rpc.rs +++ b/cli/src/modules/rpc.rs @@ -121,10 +121,20 @@ impl Rpc { // let result = rpc.get_subnetwork_call(GetSubnetworkRequest { }).await?; // self.println(&ctx, result); // } - // RpcApiOps::GetVirtualChainFromBlock => { - // let result = rpc.get_virtual_chain_from_block_call(GetVirtualChainFromBlockRequest { }).await?; - // self.println(&ctx, result); - // } + RpcApiOps::GetVirtualChainFromBlock => { + if argv.is_empty() { + return Err(Error::custom("Missing startHash argument")); + }; + let start_hash = RpcHash::from_hex(argv.remove(0).as_str())?; + let include_accepted_transaction_ids = argv.remove(0).parse::().unwrap_or_default(); + let result = rpc + .get_virtual_chain_from_block_call( + None, + GetVirtualChainFromBlockRequest { start_hash, include_accepted_transaction_ids }, + ) + .await?; + self.println(&ctx, result); + } // RpcApiOps::GetBlocks => { // let result = rpc.get_blocks_call(GetBlocksRequest { }).await?; // self.println(&ctx, result); diff --git a/components/consensusmanager/src/session.rs b/components/consensusmanager/src/session.rs index 81d589148..2643739ee 100644 --- a/components/consensusmanager/src/session.rs +++ b/components/consensusmanager/src/session.rs @@ -267,8 +267,12 @@ impl ConsensusSessionOwned { self.clone().spawn_blocking(|c| c.is_nearly_synced()).await } - pub async fn async_get_virtual_chain_from_block(&self, hash: Hash) -> ConsensusResult { - self.clone().spawn_blocking(move |c| c.get_virtual_chain_from_block(hash)).await + pub async fn async_get_virtual_chain_from_block( + &self, + low: Hash, + chain_path_added_limit: Option, + ) -> ConsensusResult { + self.clone().spawn_blocking(move |c| c.get_virtual_chain_from_block(low, chain_path_added_limit)).await } pub async fn async_get_virtual_utxos( @@ -380,8 +384,12 @@ impl ConsensusSessionOwned { /// Returns acceptance data for a set of blocks belonging to the selected parent chain. /// /// See `self::get_virtual_chain` - pub async fn async_get_blocks_acceptance_data(&self, hashes: Vec) -> ConsensusResult>> { - self.clone().spawn_blocking(move |c| c.get_blocks_acceptance_data(&hashes)).await + pub async fn async_get_blocks_acceptance_data( + &self, + hashes: Vec, + merged_blocks_limit: Option, + ) -> ConsensusResult>> { + self.clone().spawn_blocking(move |c| c.get_blocks_acceptance_data(&hashes, merged_blocks_limit)).await } pub async fn async_is_chain_block(&self, hash: Hash) -> ConsensusResult { diff --git a/consensus/core/src/api/mod.rs b/consensus/core/src/api/mod.rs index 4833c7659..91165b73d 100644 --- a/consensus/core/src/api/mod.rs +++ b/consensus/core/src/api/mod.rs @@ -157,7 +157,12 @@ pub trait ConsensusApi: Send + Sync { unimplemented!() } - fn get_virtual_chain_from_block(&self, hash: Hash) -> ConsensusResult { + /// Gets the virtual chain paths from `low` to the `sink` hash, or until `chain_path_added_limit` is reached + /// + /// Note: + /// 1) `chain_path_added_limit` will populate removed fully, and then the added chain path, up to `chain_path_added_limit` amount of hashes. + /// 1.1) use `None to impose no limit with optimized backward chain iteration, for better performance in cases where batching is not required. + fn get_virtual_chain_from_block(&self, low: Hash, chain_path_added_limit: Option) -> ConsensusResult { unimplemented!() } @@ -297,7 +302,11 @@ pub trait ConsensusApi: Send + Sync { /// Returns acceptance data for a set of blocks belonging to the selected parent chain. /// /// See `self::get_virtual_chain` - fn get_blocks_acceptance_data(&self, hashes: &[Hash]) -> ConsensusResult>> { + fn get_blocks_acceptance_data( + &self, + hashes: &[Hash], + merged_blocks_limit: Option, + ) -> ConsensusResult>> { unimplemented!() } diff --git a/consensus/src/consensus/mod.rs b/consensus/src/consensus/mod.rs index 8474a6864..1731729a3 100644 --- a/consensus/src/consensus/mod.rs +++ b/consensus/src/consensus/mod.rs @@ -607,14 +607,26 @@ impl ConsensusApi for Consensus { self.config.is_nearly_synced(compact.timestamp, compact.daa_score) } - fn get_virtual_chain_from_block(&self, hash: Hash) -> ConsensusResult { - // Calculate chain changes between the given hash and the - // sink. Note that we explicitly don't + fn get_virtual_chain_from_block(&self, low: Hash, chain_path_added_limit: Option) -> ConsensusResult { + // Calculate chain changes between the given `low` and the current sink hash (up to `limit` amount of block hashes). + // Note: + // 1) that we explicitly don't // do the calculation against the virtual itself so that we // won't later need to remove it from the result. + // 2) supplying `None` as `chain_path_added_limit` will result in the full chain path, with optimized performance. let _guard = self.pruning_lock.blocking_read(); - self.validate_block_exists(hash)?; - Ok(self.services.dag_traversal_manager.calculate_chain_path(hash, self.get_sink())) + + // Verify that the block exists + self.validate_block_exists(low)?; + + // Verify that source is on chain(block) + self.services + .reachability_service + .is_chain_ancestor_of(self.get_source(), low) + .then_some(()) + .ok_or(ConsensusError::General("the queried hash does not have source on its chain"))?; + + Ok(self.services.dag_traversal_manager.calculate_chain_path(low, self.get_sink(), chain_path_added_limit)) } /// Returns a Vec of header samples since genesis @@ -914,11 +926,35 @@ impl ConsensusApi for Consensus { self.acceptance_data_store.get(hash).unwrap_option().ok_or(ConsensusError::MissingData(hash)) } - fn get_blocks_acceptance_data(&self, hashes: &[Hash]) -> ConsensusResult>> { + fn get_blocks_acceptance_data( + &self, + hashes: &[Hash], + merged_blocks_limit: Option, + ) -> ConsensusResult>> { + // Note: merged_blocks_limit will limit after the sum of merged blocks is breached along the supplied hash's acceptance data + // and not limit the acceptance data within a queried hash. i.e. It has mergeset_size_limit granularity, this is to guarantee full acceptance data coverage. + if merged_blocks_limit.is_none() { + return hashes + .iter() + .copied() + .map(|hash| self.acceptance_data_store.get(hash).unwrap_option().ok_or(ConsensusError::MissingData(hash))) + .collect::>>(); + } + let merged_blocks_limit = merged_blocks_limit.unwrap(); // we handle `is_none`, so may unwrap. + let mut num_of_merged_blocks = 0usize; + hashes .iter() .copied() - .map(|hash| self.acceptance_data_store.get(hash).unwrap_option().ok_or(ConsensusError::MissingData(hash))) + .map_while(|hash| { + let entry = self.acceptance_data_store.get(hash).unwrap_option().ok_or(ConsensusError::MissingData(hash)); + num_of_merged_blocks += entry.as_ref().map_or(0, |entry| entry.len()); + if num_of_merged_blocks > merged_blocks_limit { + None + } else { + Some(entry) + } + }) .collect::>>() } diff --git a/consensus/src/pipeline/virtual_processor/processor.rs b/consensus/src/pipeline/virtual_processor/processor.rs index dfb7394b8..88fee97bf 100644 --- a/consensus/src/pipeline/virtual_processor/processor.rs +++ b/consensus/src/pipeline/virtual_processor/processor.rs @@ -290,7 +290,7 @@ impl VirtualStateProcessor { assert_eq!(virtual_ghostdag_data.selected_parent, new_sink); let sink_multiset = self.utxo_multisets_store.get(new_sink).unwrap(); - let chain_path = self.dag_traversal_manager.calculate_chain_path(prev_sink, new_sink); + let chain_path = self.dag_traversal_manager.calculate_chain_path(prev_sink, new_sink, None); let new_virtual_state = self .calculate_and_commit_virtual_state( virtual_read, diff --git a/consensus/src/processes/sync/mod.rs b/consensus/src/processes/sync/mod.rs index 847222968..3978913ba 100644 --- a/consensus/src/processes/sync/mod.rs +++ b/consensus/src/processes/sync/mod.rs @@ -111,7 +111,7 @@ impl< (blocks, highest_reached) } - fn find_highest_common_chain_block(&self, low: Hash, high: Hash) -> Hash { + pub fn find_highest_common_chain_block(&self, low: Hash, high: Hash) -> Hash { self.reachability_service .default_backward_chain_iterator(low) .find(|candidate| self.reachability_service.is_chain_ancestor_of(*candidate, high)) diff --git a/consensus/src/processes/traversal_manager.rs b/consensus/src/processes/traversal_manager.rs index 3ae0aef5d..23dc5c69f 100644 --- a/consensus/src/processes/traversal_manager.rs +++ b/consensus/src/processes/traversal_manager.rs @@ -31,7 +31,7 @@ impl ChainPath { + pub fn calculate_chain_path(&self, from: Hash, to: Hash, chain_path_added_limit: Option) -> ChainPath { let mut removed = Vec::new(); let mut common_ancestor = from; for current in self.reachability_service.default_backward_chain_iterator(from) { @@ -42,9 +42,20 @@ impl {} TxRemovalReason::DoubleSpend => match removed_transactions.len() { 0 => {} - 1 => warn!("Removed transaction ({}) {}{}", reason, removed_transactions[0], extra_info), - n => warn!( + 1 => debug!("Removed transaction ({}) {}{}", reason, removed_transactions[0], extra_info), + n => debug!( "Removed {} transactions ({}): {}{}", n, reason, diff --git a/rpc/grpc/core/proto/rpc.proto b/rpc/grpc/core/proto/rpc.proto index 8c3e7b3b2..e218681b6 100644 --- a/rpc/grpc/core/proto/rpc.proto +++ b/rpc/grpc/core/proto/rpc.proto @@ -374,8 +374,13 @@ message GetSubnetworkResponseMessage{ RPCError error = 1000; } -// GetVirtualChainFromBlockRequestMessage requests the virtual selected -// parent chain from some startHash to this kaspad's current virtual +/// GetVirtualChainFromBlockRequestMessage requests the virtual selected +/// parent chain from some startHash to this kaspad's current virtual +/// Note: +/// this call batches the response to: +/// a. the network's `mergeset size limit * 10` amount of added chain blocks, if `includeAcceptedTransactionIds = false` +/// b. or `mergeset size limit * 10` amount of merged blocks, if `includeAcceptedTransactionIds = true` +/// c. it does not batch the removed chain blocks, only the added ones. message GetVirtualChainFromBlockRequestMessage{ string startHash = 1; bool includeAcceptedTransactionIds = 2; diff --git a/rpc/service/src/converter/consensus.rs b/rpc/service/src/converter/consensus.rs index a5e5d3b51..c744300e5 100644 --- a/rpc/service/src/converter/consensus.rs +++ b/rpc/service/src/converter/consensus.rs @@ -162,8 +162,9 @@ impl ConsensusConverter { &self, consensus: &ConsensusProxy, chain_path: &ChainPath, + merged_blocks_limit: Option, ) -> RpcResult> { - let acceptance_data = consensus.async_get_blocks_acceptance_data(chain_path.added.clone()).await.unwrap(); + let acceptance_data = consensus.async_get_blocks_acceptance_data(chain_path.added.clone(), merged_blocks_limit).await.unwrap(); Ok(chain_path .added .iter() diff --git a/rpc/service/src/service.rs b/rpc/service/src/service.rs index 2c22fd6bb..bdc2a9541 100644 --- a/rpc/service/src/service.rs +++ b/rpc/service/src/service.rs @@ -539,7 +539,7 @@ NOTE: This error usually indicates an RPC conversion error between the node and ) -> RpcResult { let allow_orphan = self.config.unsafe_rpc && request.allow_orphan; if !self.config.unsafe_rpc && request.allow_orphan { - warn!("SubmitTransaction RPC command called with AllowOrphan enabled while node in safe RPC mode -- switching to ForbidOrphan."); + debug!("SubmitTransaction RPC command called with AllowOrphan enabled while node in safe RPC mode -- switching to ForbidOrphan."); } let transaction: Transaction = request.transaction.try_into()?; @@ -609,13 +609,26 @@ NOTE: This error usually indicates an RPC conversion error between the node and request: GetVirtualChainFromBlockRequest, ) -> RpcResult { let session = self.consensus_manager.consensus().session().await; - let virtual_chain = session.async_get_virtual_chain_from_block(request.start_hash).await?; + + // batch_size is set to 10 times the mergeset_size_limit. + // this means batch_size is 2480 on 10 bps, and 1800 on mainnet. + // this bounds by number of merged blocks, if include_accepted_transactions = true + // else it returns the batch_size amount on pure chain blocks. + // Note: batch_size does not bound removed chain blocks, only added chain blocks. + let batch_size = (self.config.mergeset_size_limit * 10) as usize; + let mut virtual_chain_batch = session.async_get_virtual_chain_from_block(request.start_hash, Some(batch_size)).await?; let accepted_transaction_ids = if request.include_accepted_transaction_ids { - self.consensus_converter.get_virtual_chain_accepted_transaction_ids(&session, &virtual_chain).await? + let accepted_transaction_ids = self + .consensus_converter + .get_virtual_chain_accepted_transaction_ids(&session, &virtual_chain_batch, Some(batch_size)) + .await?; + // bound added to the length of the accepted transaction ids, which is bounded by merged blocks + virtual_chain_batch.added = virtual_chain_batch.added[..accepted_transaction_ids.len()].to_vec(); + accepted_transaction_ids } else { vec![] }; - Ok(GetVirtualChainFromBlockResponse::new(virtual_chain.removed, virtual_chain.added, accepted_transaction_ids)) + Ok(GetVirtualChainFromBlockResponse::new(virtual_chain_batch.removed, virtual_chain_batch.added, accepted_transaction_ids)) } async fn get_block_count_call( diff --git a/testing/integration/src/daemon_integration_tests.rs b/testing/integration/src/daemon_integration_tests.rs index a92320294..460cf049c 100644 --- a/testing/integration/src/daemon_integration_tests.rs +++ b/testing/integration/src/daemon_integration_tests.rs @@ -106,7 +106,13 @@ async fn daemon_mining_test() { assert_eq!(dag_info.sink, last_block_hash.unwrap()); // Check that acceptance data contains the expected coinbase tx ids - let vc = rpc_client2.get_virtual_chain_from_block(kaspa_consensus::params::SIMNET_GENESIS.hash, true).await.unwrap(); + let vc = rpc_client2 + .get_virtual_chain_from_block( + kaspa_consensus::params::SIMNET_GENESIS.hash, // + true, + ) + .await + .unwrap(); assert_eq!(vc.removed_chain_block_hashes.len(), 0); assert_eq!(vc.added_chain_block_hashes.len(), 10); assert_eq!(vc.accepted_transaction_ids.len(), 10);