From bd2624572a65019263ef3a71dd15be751c938207 Mon Sep 17 00:00:00 2001 From: "hinto.janai" Date: Thu, 12 Dec 2024 11:47:44 -0500 Subject: [PATCH 01/16] add `/tests` --- Cargo.lock | 227 +++++++++++++++++++++++++++++++++ Cargo.toml | 3 + tests/monero-serai/Cargo.toml | 17 +++ tests/monero-serai/src/main.rs | 72 +++++++++++ tests/monero-serai/src/rpc.rs | 187 +++++++++++++++++++++++++++ 5 files changed, 506 insertions(+) create mode 100644 tests/monero-serai/Cargo.toml create mode 100644 tests/monero-serai/src/main.rs create mode 100644 tests/monero-serai/src/rpc.rs diff --git a/Cargo.lock b/Cargo.lock index c8701f35..9d4123c8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1363,6 +1363,15 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -1431,6 +1440,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -1779,6 +1803,22 @@ dependencies = [ "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "hyper-util" version = "0.1.10" @@ -1841,6 +1881,12 @@ dependencies = [ "hashbrown 0.15.0", ] +[[package]] +name = "ipnet" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" + [[package]] name = "is-terminal" version = "0.4.12" @@ -2186,6 +2232,23 @@ dependencies = [ "tokio", ] +[[package]] +name = "native-tls" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -2227,12 +2290,50 @@ version = "11.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" +[[package]] +name = "openssl" +version = "0.10.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" +dependencies = [ + "bitflags 2.6.0", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "openssl-probe" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "openssl-sys" +version = "0.9.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "option-ext" version = "0.2.0" @@ -2364,6 +2465,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" + [[package]] name = "plotters" version = "0.3.6" @@ -2676,6 +2783,49 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "reqwest" +version = "0.12.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-tls", + "hyper-util", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper 1.0.1", + "system-configuration", + "tokio", + "tokio-native-tls", + "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "windows-registry", +] + [[package]] name = "ring" version = "0.17.8" @@ -3104,6 +3254,9 @@ name = "sync_wrapper" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +dependencies = [ + "futures-core", +] [[package]] name = "synchronoise" @@ -3114,6 +3267,27 @@ dependencies = [ "crossbeam-queue", ] +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags 2.6.0", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tap" version = "1.0.1" @@ -3143,6 +3317,20 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "tests-monero-serai" +version = "0.1.0" +dependencies = [ + "futures", + "hex", + "monero-serai", + "rayon", + "reqwest", + "serde", + "serde_json", + "tokio", +] + [[package]] name = "thiserror" version = "1.0.66" @@ -3233,6 +3421,16 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.0" @@ -3520,6 +3718,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" @@ -3586,6 +3790,18 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7ec4f8827a71586374db3e87abdb5a2bb3a15afed140221307c3ec06b1f63b" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.95" @@ -3719,6 +3935,17 @@ dependencies = [ "syn", ] +[[package]] +name = "windows-registry" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" +dependencies = [ + "windows-result", + "windows-strings", + "windows-targets 0.52.6", +] + [[package]] name = "windows-result" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index 18130577..3f0257d1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,6 +52,9 @@ members = [ "pruning", "test-utils", "types", + + # Tests + "tests/monero-serai", ] [profile.release] diff --git a/tests/monero-serai/Cargo.toml b/tests/monero-serai/Cargo.toml new file mode 100644 index 00000000..8b246a78 --- /dev/null +++ b/tests/monero-serai/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "tests-monero-serai" +version = "0.1.0" +edition = "2021" + +[dependencies] +monero-serai = { workspace = true } +hex = { workspace = true, features = ["serde", "std"] } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true, features = ["std"] } +tokio = { workspace = true, features = ["full"] } +reqwest = { version = "0.12", features = ["json"] } +rayon = { workspace = true } +futures = { workspace = true, features = ["std"] } + +[lints] +workspace = true diff --git a/tests/monero-serai/src/main.rs b/tests/monero-serai/src/main.rs new file mode 100644 index 00000000..370be33d --- /dev/null +++ b/tests/monero-serai/src/main.rs @@ -0,0 +1,72 @@ +use std::{ + sync::atomic::{AtomicUsize, Ordering}, + time::{Duration, Instant}, +}; + +mod rpc; + +pub static TESTED_BLOCK_COUNT: AtomicUsize = AtomicUsize::new(0); + +#[tokio::main] +async fn main() { + let now = Instant::now(); + + let rpc_node_url = if let Ok(url) = std::env::var("RPC_NODE_URL") { + url + } else { + "http://127.0.0.1:18081/json_rpc".to_string() + }; + println!("rpc_node_url: {rpc_node_url}"); + + let top_height = rpc::RpcClient::top_height(rpc_node_url.clone()).await; + println!("top_height: {top_height}"); + assert!(top_height > 3301441, "node is behind"); + + let ranges = (0..top_height) + .collect::>() + .chunks(100_000) + .map(<[usize]>::to_vec) + .collect::>>(); + + println!("ranges: "); + for range in &ranges { + println!("[{}..{}]", range.first().unwrap(), range.last().unwrap()); + } + + let rpc_client = rpc::RpcClient::new(rpc_node_url); + + let iter = ranges.into_iter().map(move |range| { + let c = rpc_client.clone(); + async move { + tokio::task::spawn_blocking(move || async move { + c.get_block_test_batch(range.into_iter().collect()).await; + }) + .await + .unwrap() + .await; + } + }); + + std::thread::spawn(move || { + let mut count = 0; + + #[expect(clippy::cast_precision_loss)] + while count != top_height { + let c = TESTED_BLOCK_COUNT.load(Ordering::Acquire); + count = c; + + println!( + "blocks processed ... {c} ({:.2}%)", + (c as f64 / top_height as f64) * 100.0 + ); + + std::thread::sleep(Duration::from_millis(250)); + } + + println!("finished all blocks, took: {}s", now.elapsed().as_secs()); + std::process::exit(0); + }); + + futures::future::join_all(iter).await; + std::thread::park(); +} diff --git a/tests/monero-serai/src/rpc.rs b/tests/monero-serai/src/rpc.rs new file mode 100644 index 00000000..66074b1b --- /dev/null +++ b/tests/monero-serai/src/rpc.rs @@ -0,0 +1,187 @@ +use std::{collections::BTreeSet, sync::atomic::Ordering}; + +use hex::serde::deserialize; +use monero_serai::block::Block; +use reqwest::{ + header::{HeaderMap, HeaderValue}, + Client, ClientBuilder, +}; +use serde::Deserialize; +use serde_json::json; + +use crate::TESTED_BLOCK_COUNT; + +#[derive(Debug, Clone, Deserialize)] +pub(crate) struct BlockHeader { + #[serde(deserialize_with = "deserialize")] + pub hash: Vec, + #[serde(deserialize_with = "deserialize")] + pub miner_tx_hash: Vec, + #[serde(deserialize_with = "deserialize")] + pub prev_hash: Vec, + + pub block_weight: usize, + pub height: usize, + pub major_version: u8, + pub minor_version: u8, + pub nonce: u32, + pub num_txes: usize, + pub reward: u64, + pub timestamp: u64, +} + +#[derive(Debug, Clone)] +pub(crate) struct RpcClient { + client: Client, + rpc_node_url: String, +} + +impl RpcClient { + pub(crate) fn new(rpc_node_url: String) -> Self { + let headers = { + let mut h = HeaderMap::new(); + h.insert("Content-Type", HeaderValue::from_static("application/json")); + h + }; + + let client = ClientBuilder::new() + .default_headers(headers) + .build() + .unwrap(); + + Self { + client, + rpc_node_url, + } + } + + pub(crate) async fn top_height(rpc_node_url: String) -> usize { + #[derive(Debug, Clone, Deserialize)] + struct JsonRpcResponse { + result: GetLastBlockHeaderResponse, + } + + #[derive(Debug, Clone, Deserialize)] + pub(crate) struct GetLastBlockHeaderResponse { + pub block_header: BlockHeader, + } + + let this = Self::new(rpc_node_url); + + let request = json!({ + "jsonrpc": "2.0", + "id": 0, + "method": "get_last_block_header", + "params": {} + }); + + this.client + .get(this.rpc_node_url) + .json(&request) + .send() + .await + .unwrap() + .json::() + .await + .unwrap() + .result + .block_header + .height + } + + pub(crate) async fn get_block_test_batch(&self, heights: BTreeSet) { + #[derive(Debug, Clone, Deserialize)] + struct JsonRpcResponse { + result: GetBlockResponse, + } + + #[derive(Debug, Clone, Deserialize)] + pub(crate) struct GetBlockResponse { + #[serde(deserialize_with = "deserialize")] + pub blob: Vec, + pub block_header: BlockHeader, + } + + let tasks = heights.into_iter().map(|height| { + let request = json!({ + "jsonrpc": "2.0", + "id": 0, + "method": "get_block", + "params": {"height": height} + }); + + let task = + tokio::task::spawn(self.client.get(&self.rpc_node_url).json(&request).send()); + + (height, task) + }); + + for (height, task) in tasks { + let resp = task + .await + .unwrap() + .unwrap() + .json::() + .await + .unwrap() + .result; + + rayon::spawn(move || { + let info = format!("\nheight: {height}\nresponse: {resp:#?}"); + + // Test block deserialization. + let block = match Block::read(&mut resp.blob.as_slice()) { + Ok(b) => b, + Err(e) => panic!("{e:?}\n{info}"), + }; + + // Test block properties. + assert_eq!(resp.blob, block.serialize(), "{info}"); + + assert!( + !block.miner_transaction.prefix().outputs.is_empty(), + "miner_tx has no outputs\n{info}" + ); + + let block_reward = block + .miner_transaction + .prefix() + .outputs + .iter() + .map(|o| o.amount.unwrap()) + .sum::(); + assert_ne!(block_reward, 0, "block reward is 0\n{info}"); + + // Test fields are correct. + let BlockHeader { + block_weight, + hash, + height, + major_version, + minor_version, + miner_tx_hash, + nonce, + num_txes, + prev_hash, + reward, + timestamp, + } = resp.block_header; + + assert_ne!(block_weight, 0, "{info}"); // TODO: test this + assert_ne!(block.miner_transaction.weight(), 0, "{info}"); // TODO: test this + assert_eq!(hash, block.hash(), "{info}"); + assert_eq!(height, block.number().unwrap(), "{info}"); + assert_eq!(major_version, block.header.hardfork_version, "{info}"); + assert_eq!(minor_version, block.header.hardfork_signal, "{info}"); + assert_eq!(miner_tx_hash, block.miner_transaction.hash(), "{info}"); + assert_eq!(nonce, block.header.nonce, "{info}"); + assert_eq!(num_txes, block.transactions.len(), "{info}"); + assert_eq!(prev_hash, block.header.previous, "{info}"); + assert_eq!(reward, block_reward, "{info}"); + assert_eq!(timestamp, block.header.timestamp, "{info}"); + + TESTED_BLOCK_COUNT.fetch_add(1, Ordering::Release); + }); + } + } +} From 1d1cba4f63bc4f0d7d950e5e44f41863e8863786 Mon Sep 17 00:00:00 2001 From: "hinto.janai" Date: Thu, 12 Dec 2024 14:56:32 -0500 Subject: [PATCH 02/16] add `tests/pow` --- Cargo.lock | 18 +++ Cargo.toml | 1 + tests/monero-serai/src/main.rs | 2 +- tests/pow/Cargo.toml | 22 +++ tests/pow/src/main.rs | 32 +++++ tests/pow/src/rpc.rs | 254 +++++++++++++++++++++++++++++++++ 6 files changed, 328 insertions(+), 1 deletion(-) create mode 100644 tests/pow/Cargo.toml create mode 100644 tests/pow/src/main.rs create mode 100644 tests/pow/src/rpc.rs diff --git a/Cargo.lock b/Cargo.lock index 9d4123c8..3e96b6a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3331,6 +3331,24 @@ dependencies = [ "tokio", ] +[[package]] +name = "tests-pow" +version = "0.1.0" +dependencies = [ + "cuprate-consensus-rules", + "cuprate-cryptonight", + "function_name", + "hex", + "monero-serai", + "randomx-rs", + "rayon", + "reqwest", + "serde", + "serde_json", + "thread_local", + "tokio", +] + [[package]] name = "thiserror" version = "1.0.66" diff --git a/Cargo.toml b/Cargo.toml index 3f0257d1..b9424371 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,6 +54,7 @@ members = [ "types", # Tests + "tests/pow", "tests/monero-serai", ] diff --git a/tests/monero-serai/src/main.rs b/tests/monero-serai/src/main.rs index 370be33d..e5fbc309 100644 --- a/tests/monero-serai/src/main.rs +++ b/tests/monero-serai/src/main.rs @@ -56,7 +56,7 @@ async fn main() { count = c; println!( - "blocks processed ... {c} ({:.2}%)", + "blocks processed ... {c}/{top_height} ({:.2}%)", (c as f64 / top_height as f64) * 100.0 ); diff --git a/tests/pow/Cargo.toml b/tests/pow/Cargo.toml new file mode 100644 index 00000000..4b965dd4 --- /dev/null +++ b/tests/pow/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "tests-pow" +version = "0.1.0" +edition = "2021" + +[dependencies] +cuprate-cryptonight = { workspace = true } +cuprate-consensus-rules = { workspace = true } + +function_name = { workspace = true } +thread_local = { workspace = true } +monero-serai = { workspace = true } +hex = { workspace = true, features = ["serde", "std"] } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true, features = ["std"] } +tokio = { workspace = true, features = ["full"] } +reqwest = { version = "0.12", features = ["json"] } +rayon = { workspace = true } +randomx-rs = { workspace = true } + +[lints] +workspace = true diff --git a/tests/pow/src/main.rs b/tests/pow/src/main.rs new file mode 100644 index 00000000..dbb9b9c8 --- /dev/null +++ b/tests/pow/src/main.rs @@ -0,0 +1,32 @@ +use std::{ + sync::atomic::{AtomicUsize, Ordering}, + time::{Duration, Instant}, +}; + +mod rpc; + +pub static TESTED_BLOCK_COUNT: AtomicUsize = AtomicUsize::new(0); + +#[tokio::main] +async fn main() { + let now = Instant::now(); + + let rpc_node_url = if let Ok(url) = std::env::var("RPC_NODE_URL") { + url + } else { + "http://127.0.0.1:18081/json_rpc".to_string() + }; + println!("rpc_node_url: {rpc_node_url}"); + + let rpc_client = rpc::RpcClient::new(rpc_node_url).await; + + tokio::join!( + rpc_client.cryptonight_v0(), + rpc_client.cryptonight_v1(), + rpc_client.cryptonight_v2(), + rpc_client.cryptonight_r(), + rpc_client.randomx(), + ); + + println!("finished all PoW, took: {}s", now.elapsed().as_secs()); +} diff --git a/tests/pow/src/rpc.rs b/tests/pow/src/rpc.rs new file mode 100644 index 00000000..e4a2bef2 --- /dev/null +++ b/tests/pow/src/rpc.rs @@ -0,0 +1,254 @@ +use std::{ + collections::BTreeMap, + ops::Range, + sync::{atomic::Ordering, Mutex}, +}; + +use function_name::named; +use hex::serde::deserialize; +use monero_serai::block::Block; +use randomx_rs::{RandomXCache, RandomXFlag, RandomXVM}; +use reqwest::{ + header::{HeaderMap, HeaderValue}, + Client, ClientBuilder, +}; +use serde::Deserialize; +use serde_json::{json, Value}; +use thread_local::ThreadLocal; + +use crate::TESTED_BLOCK_COUNT; + +#[derive(Debug, Clone, Deserialize)] +struct JsonRpcResponse { + result: GetBlockResponse, +} + +#[derive(Debug, Clone, Deserialize)] +struct GetBlockResponse { + #[serde(deserialize_with = "deserialize")] + pub blob: Vec, + pub block_header: BlockHeader, +} + +#[derive(Debug, Clone, Deserialize)] +struct BlockHeader { + #[serde(deserialize_with = "deserialize")] + pub pow_hash: Vec, + #[serde(deserialize_with = "deserialize")] + pub hash: Vec, +} + +#[derive(Debug, Clone)] +pub(crate) struct RpcClient { + client: Client, + rpc_node_url: String, + top_height: usize, +} + +impl RpcClient { + pub(crate) async fn new(rpc_node_url: String) -> Self { + let headers = { + let mut h = HeaderMap::new(); + h.insert("Content-Type", HeaderValue::from_static("application/json")); + h + }; + + let client = ClientBuilder::new() + .default_headers(headers) + .build() + .unwrap(); + + let request = json!({ + "jsonrpc": "2.0", + "id": 0, + "method": "get_last_block_header", + "params": {} + }); + + let top_height = client + .get(&rpc_node_url) + .json(&request) + .send() + .await + .unwrap() + .json::() + .await + .unwrap() + .get("result") + .unwrap() + .get("block_header") + .unwrap() + .get("height") + .unwrap() + .as_u64() + .unwrap() + .try_into() + .unwrap(); + + println!("top_height: {top_height}"); + assert!(top_height > 3301441, "node is behind"); + + Self { + client, + rpc_node_url, + top_height, + } + } + + async fn get_block(&self, height: usize) -> GetBlockResponse { + let request = json!({ + "jsonrpc": "2.0", + "id": 0, + "method": "get_block", + "params": {"height": height, "fill_pow_hash": true} + }); + + tokio::task::spawn(self.client.get(&self.rpc_node_url).json(&request).send()) + .await + .unwrap() + .unwrap() + .json::() + .await + .unwrap() + .result + } + + async fn test( + &self, + range: Range, + hash: impl Fn(Vec, u64, u64, [u8; 32]) -> [u8; 32] + Send + Sync + 'static + Copy, + name: &'static str, + ) { + let tasks = range.map(|height| { + let task = self.get_block(height); + (height, task) + }); + + for (height, task) in tasks { + let result = task.await; + + let (seed_height, seed_hash) = if RANDOMX { + let seed_height = cuprate_consensus_rules::blocks::randomx_seed_height(height); + + let seed_hash: [u8; 32] = self + .get_block(seed_height) + .await + .block_header + .hash + .try_into() + .unwrap(); + + (seed_height, seed_hash) + } else { + (0, [0; 32]) + }; + + let top_height = self.top_height; + + #[expect(clippy::cast_precision_loss)] + rayon::spawn(move || { + let GetBlockResponse { blob, block_header } = result; + let header = block_header; + + let block = match Block::read(&mut blob.as_slice()) { + Ok(b) => b, + Err(e) => panic!("{e:?}\nblob: {blob:?}, header: {header:?}"), + }; + + let pow_hash = hash( + block.serialize_pow_hash(), + height.try_into().unwrap(), + seed_height.try_into().unwrap(), + seed_hash, + ); + + assert_eq!( + header.pow_hash, pow_hash, + "\nheight: {height}\nheader: {header:#?}\nblock: {block:#?}" + ); + + let count = TESTED_BLOCK_COUNT.fetch_add(1, Ordering::Release); + + let hex_header = hex::encode(header.pow_hash); + let hex_hash = hex::encode(pow_hash); + let percent = (count as f64 / top_height as f64) * 100.0; + + println!( + "progress | {count}/{top_height} ({percent:.2}%) +height | {height} +algo | {name} +header | {hex_header} +hash | {hex_hash}\n" + ); + }); + } + } + + #[named] + pub(crate) async fn cryptonight_v0(&self) { + self.test::( + 0..1546000, + |b, _, _, _| cuprate_cryptonight::cryptonight_hash_v0(&b), + function_name!(), + ) + .await; + } + + #[named] + pub(crate) async fn cryptonight_v1(&self) { + self.test::( + 1546000..1685555, + |b, _, _, _| cuprate_cryptonight::cryptonight_hash_v1(&b).unwrap(), + function_name!(), + ) + .await; + } + + #[named] + pub(crate) async fn cryptonight_v2(&self) { + self.test::( + 1685555..1788000, + |b, _, _, _| cuprate_cryptonight::cryptonight_hash_v2(&b), + function_name!(), + ) + .await; + } + + #[named] + pub(crate) async fn cryptonight_r(&self) { + self.test::( + 1788000..1978433, + |b, h, _, _| cuprate_cryptonight::cryptonight_hash_r(&b, h), + function_name!(), + ) + .await; + } + + #[named] + pub(crate) async fn randomx(&self) { + #[expect(clippy::significant_drop_tightening)] + let function = move |bytes: Vec, _, seed_height, seed_hash: [u8; 32]| { + static RANDOMX_VM: ThreadLocal>> = ThreadLocal::new(); + + let mut thread_local = RANDOMX_VM + .get_or(|| Mutex::new(BTreeMap::new())) + .lock() + .unwrap(); + + let randomx_vm = thread_local.entry(seed_height).or_insert_with(|| { + let flag = RandomXFlag::get_recommended_flags(); + let cache = RandomXCache::new(flag, &seed_hash).unwrap(); + RandomXVM::new(flag, Some(cache), None).unwrap() + }); + + randomx_vm + .calculate_hash(&bytes) + .unwrap() + .try_into() + .unwrap() + }; + + self.test::(1978433..self.top_height, function, function_name!()) + .await; + } +} From 0a44af4b8cc421cb03b032c33758edadf4c67123 Mon Sep 17 00:00:00 2001 From: "hinto.janai" Date: Thu, 12 Dec 2024 17:49:25 -0500 Subject: [PATCH 03/16] verify all txs in blocks --- tests/monero-serai/src/main.rs | 43 ++++----- tests/monero-serai/src/rpc.rs | 163 +++++++++++++++++++++++++++------ tests/pow/src/main.rs | 34 ++++--- tests/pow/src/rpc.rs | 14 +-- 4 files changed, 182 insertions(+), 72 deletions(-) diff --git a/tests/monero-serai/src/main.rs b/tests/monero-serai/src/main.rs index e5fbc309..6bbf4701 100644 --- a/tests/monero-serai/src/main.rs +++ b/tests/monero-serai/src/main.rs @@ -1,4 +1,5 @@ use std::{ + io::Write, sync::atomic::{AtomicUsize, Ordering}, time::{Duration, Instant}, }; @@ -6,21 +7,21 @@ use std::{ mod rpc; pub static TESTED_BLOCK_COUNT: AtomicUsize = AtomicUsize::new(0); +pub static TESTED_TX_COUNT: AtomicUsize = AtomicUsize::new(0); #[tokio::main] async fn main() { let now = Instant::now(); - let rpc_node_url = if let Ok(url) = std::env::var("RPC_NODE_URL") { + let rpc_url = if let Ok(url) = std::env::var("RPC_URL") { url } else { - "http://127.0.0.1:18081/json_rpc".to_string() + "http://127.0.0.1:18081".to_string() }; - println!("rpc_node_url: {rpc_node_url}"); + println!("rpc_url: {rpc_url}"); - let top_height = rpc::RpcClient::top_height(rpc_node_url.clone()).await; - println!("top_height: {top_height}"); - assert!(top_height > 3301441, "node is behind"); + let client = rpc::RpcClient::new(rpc_url).await; + let top_height = client.top_height; let ranges = (0..top_height) .collect::>() @@ -33,10 +34,8 @@ async fn main() { println!("[{}..{}]", range.first().unwrap(), range.last().unwrap()); } - let rpc_client = rpc::RpcClient::new(rpc_node_url); - let iter = ranges.into_iter().map(move |range| { - let c = rpc_client.clone(); + let c = client.clone(); async move { tokio::task::spawn_blocking(move || async move { c.get_block_test_batch(range.into_iter().collect()).await; @@ -47,26 +46,20 @@ async fn main() { } }); - std::thread::spawn(move || { - let mut count = 0; + futures::future::join_all(iter).await; - #[expect(clippy::cast_precision_loss)] - while count != top_height { - let c = TESTED_BLOCK_COUNT.load(Ordering::Acquire); - count = c; + loop { + let block_count = TESTED_BLOCK_COUNT.load(Ordering::Acquire); + let tx_count = TESTED_TX_COUNT.load(Ordering::Acquire); + if top_height == block_count { println!( - "blocks processed ... {c}/{top_height} ({:.2}%)", - (c as f64 / top_height as f64) * 100.0 + "finished processing: blocks: {block_count}/{top_height}, txs: {tx_count}, took {}s", + now.elapsed().as_secs() ); - - std::thread::sleep(Duration::from_millis(250)); + std::process::exit(0); } - println!("finished all blocks, took: {}s", now.elapsed().as_secs()); - std::process::exit(0); - }); - - futures::future::join_all(iter).await; - std::thread::park(); + std::thread::sleep(Duration::from_secs(1)); + } } diff --git a/tests/monero-serai/src/rpc.rs b/tests/monero-serai/src/rpc.rs index 66074b1b..c08aa857 100644 --- a/tests/monero-serai/src/rpc.rs +++ b/tests/monero-serai/src/rpc.rs @@ -1,7 +1,8 @@ use std::{collections::BTreeSet, sync::atomic::Ordering}; use hex::serde::deserialize; -use monero_serai::block::Block; +use monero_serai::{block::Block, transaction::Transaction}; +use rayon::iter::{IndexedParallelIterator, IntoParallelIterator, ParallelIterator}; use reqwest::{ header::{HeaderMap, HeaderValue}, Client, ClientBuilder, @@ -9,7 +10,7 @@ use reqwest::{ use serde::Deserialize; use serde_json::json; -use crate::TESTED_BLOCK_COUNT; +use crate::{TESTED_BLOCK_COUNT, TESTED_TX_COUNT}; #[derive(Debug, Clone, Deserialize)] pub(crate) struct BlockHeader { @@ -33,11 +34,12 @@ pub(crate) struct BlockHeader { #[derive(Debug, Clone)] pub(crate) struct RpcClient { client: Client, - rpc_node_url: String, + rpc_url: String, + pub top_height: usize, } impl RpcClient { - pub(crate) fn new(rpc_node_url: String) -> Self { + pub(crate) async fn new(rpc_url: String) -> Self { let headers = { let mut h = HeaderMap::new(); h.insert("Content-Type", HeaderValue::from_static("application/json")); @@ -49,13 +51,6 @@ impl RpcClient { .build() .unwrap(); - Self { - client, - rpc_node_url, - } - } - - pub(crate) async fn top_height(rpc_node_url: String) -> usize { #[derive(Debug, Clone, Deserialize)] struct JsonRpcResponse { result: GetLastBlockHeaderResponse, @@ -66,8 +61,6 @@ impl RpcClient { pub block_header: BlockHeader, } - let this = Self::new(rpc_node_url); - let request = json!({ "jsonrpc": "2.0", "id": 0, @@ -75,8 +68,8 @@ impl RpcClient { "params": {} }); - this.client - .get(this.rpc_node_url) + let top_height = client + .get(format!("{rpc_url}/json_rpc")) .json(&request) .send() .await @@ -86,7 +79,67 @@ impl RpcClient { .unwrap() .result .block_header - .height + .height; + + println!("top_height: {top_height}"); + assert!(top_height > 3301441, "node is behind"); + + Self { + client, + rpc_url, + top_height, + } + } + + async fn get_transactions(&self, tx_hashes: Vec<[u8; 32]>) -> Vec<(Transaction, Vec)> { + assert!(!tx_hashes.is_empty()); + + #[derive(Debug, Clone, Deserialize)] + pub(crate) struct GetTransactionsResponse { + pub txs: Vec, + } + + #[derive(Debug, Clone, Deserialize)] + pub(crate) struct Tx { + pub as_hex: String, + pub pruned_as_hex: String, + } + + let url = format!("{}/get_transactions", self.rpc_url); + + let txs_hashes = tx_hashes + .into_iter() + .map(hex::encode) + .collect::>(); + + let request = json!({ + "txs_hashes": txs_hashes, + }); + + let txs = self + .client + .get(&url) + .json(&request) + .send() + .await + .unwrap() + .json::() + .await + .unwrap() + .txs; + + txs.into_par_iter() + .map(|r| { + let blob = hex::decode(if r.as_hex.is_empty() { + r.pruned_as_hex + } else { + r.as_hex + }) + .unwrap(); + + (Transaction::read(&mut blob.as_slice()).unwrap(), blob) + }) + .collect() } pub(crate) async fn get_block_test_batch(&self, heights: BTreeSet) { @@ -103,6 +156,8 @@ impl RpcClient { } let tasks = heights.into_iter().map(|height| { + let json_rpc_url = format!("{}/json_rpc", self.rpc_url); + let request = json!({ "jsonrpc": "2.0", "id": 0, @@ -110,8 +165,7 @@ impl RpcClient { "params": {"height": height} }); - let task = - tokio::task::spawn(self.client.get(&self.rpc_node_url).json(&request).send()); + let task = tokio::task::spawn(self.client.get(&json_rpc_url).json(&request).send()); (height, task) }); @@ -126,15 +180,24 @@ impl RpcClient { .unwrap() .result; - rayon::spawn(move || { - let info = format!("\nheight: {height}\nresponse: {resp:#?}"); + let info = format!("\nheight: {height}\nresponse: {resp:#?}"); - // Test block deserialization. - let block = match Block::read(&mut resp.blob.as_slice()) { - Ok(b) => b, - Err(e) => panic!("{e:?}\n{info}"), - }; + // Test block deserialization. + let block = match Block::read(&mut resp.blob.as_slice()) { + Ok(b) => b, + Err(e) => panic!("{e:?}\n{info}"), + }; + // Fetch all transactions. + let mut tx_hashes = vec![block.miner_transaction.hash()]; + tx_hashes.extend(block.transactions.iter()); + let txs = self.get_transactions(tx_hashes.clone()).await; + assert_eq!(tx_hashes.len(), txs.len()); + + let top_height = self.top_height; + + #[expect(clippy::cast_precision_loss)] + rayon::spawn(move || { // Test block properties. assert_eq!(resp.blob, block.serialize(), "{info}"); @@ -167,8 +230,21 @@ impl RpcClient { timestamp, } = resp.block_header; - assert_ne!(block_weight, 0, "{info}"); // TODO: test this - assert_ne!(block.miner_transaction.weight(), 0, "{info}"); // TODO: test this + let total_block_weight = txs.iter().map(|(tx, _)| tx.weight()).sum::(); + + // Test transaction properties. + txs.into_par_iter() + .zip(tx_hashes) + .for_each(|((tx, blob), hash)| { + assert_eq!(hash, tx.hash(), "{info}, tx: {tx:#?}"); + assert_ne!(tx.weight(), 0, "{info}, tx: {tx:#?}"); + assert!(!tx.prefix().inputs.is_empty(), "{info}, tx: {tx:#?}"); + assert_eq!(blob, tx.serialize(), "{info}, tx: {tx:#?}"); + assert!(matches!(tx.version(), 1 | 2), "{info}, tx: {tx:#?}"); + }); + + assert_eq!(block_weight, total_block_weight, "{info}"); + assert_ne!(block.miner_transaction.weight(), 0, "{info}"); assert_eq!(hash, block.hash(), "{info}"); assert_eq!(height, block.number().unwrap(), "{info}"); assert_eq!(major_version, block.header.hardfork_version, "{info}"); @@ -180,7 +256,38 @@ impl RpcClient { assert_eq!(reward, block_reward, "{info}"); assert_eq!(timestamp, block.header.timestamp, "{info}"); - TESTED_BLOCK_COUNT.fetch_add(1, Ordering::Release); + let block_count = TESTED_BLOCK_COUNT.fetch_add(1, Ordering::Release) + 1; + let tx_count = TESTED_TX_COUNT.fetch_add(num_txes, Ordering::Release) + 1; + let percent = (block_count as f64 / top_height as f64) * 100.0; + + println!( + "block_count | {block_count}/{top_height} ({percent:.2}%) +tx_count | {tx_count} +hash | {} +miner_tx_hash | {} +prev_hash | {} +reward | {} +timestamp | {} +nonce | {} +height | {} +block_weight | {} +miner_tx_weight | {} +major_version | {} +minor_version | {} +num_txes | {}\n", + hex::encode(hash), + hex::encode(miner_tx_hash), + hex::encode(prev_hash), + reward, + timestamp, + nonce, + height, + block_weight, + block.miner_transaction.weight(), + major_version, + minor_version, + num_txes, + ); }); } } diff --git a/tests/pow/src/main.rs b/tests/pow/src/main.rs index dbb9b9c8..9ac4d31e 100644 --- a/tests/pow/src/main.rs +++ b/tests/pow/src/main.rs @@ -1,32 +1,42 @@ +mod rpc; + use std::{ sync::atomic::{AtomicUsize, Ordering}, time::{Duration, Instant}, }; -mod rpc; - pub static TESTED_BLOCK_COUNT: AtomicUsize = AtomicUsize::new(0); #[tokio::main] async fn main() { let now = Instant::now(); - let rpc_node_url = if let Ok(url) = std::env::var("RPC_NODE_URL") { + let rpc_url = if let Ok(url) = std::env::var("RPC_URL") { url } else { - "http://127.0.0.1:18081/json_rpc".to_string() + "http://127.0.0.1:18081".to_string() }; - println!("rpc_node_url: {rpc_node_url}"); + println!("rpc_url: {rpc_url}"); - let rpc_client = rpc::RpcClient::new(rpc_node_url).await; + let client = rpc::RpcClient::new(rpc_url).await; + let top_height = client.top_height; tokio::join!( - rpc_client.cryptonight_v0(), - rpc_client.cryptonight_v1(), - rpc_client.cryptonight_v2(), - rpc_client.cryptonight_r(), - rpc_client.randomx(), + client.cryptonight_v0(), + client.cryptonight_v1(), + client.cryptonight_v2(), + client.cryptonight_r(), + client.randomx(), ); - println!("finished all PoW, took: {}s", now.elapsed().as_secs()); + loop { + let count = TESTED_BLOCK_COUNT.load(Ordering::Acquire); + + if top_height == count { + println!("finished all PoW, took {}s", now.elapsed().as_secs()); + std::process::exit(0); + } + + std::thread::sleep(Duration::from_secs(1)); + } } diff --git a/tests/pow/src/rpc.rs b/tests/pow/src/rpc.rs index e4a2bef2..426c525e 100644 --- a/tests/pow/src/rpc.rs +++ b/tests/pow/src/rpc.rs @@ -41,12 +41,12 @@ struct BlockHeader { #[derive(Debug, Clone)] pub(crate) struct RpcClient { client: Client, - rpc_node_url: String, - top_height: usize, + rpc_url: String, + pub top_height: usize, } impl RpcClient { - pub(crate) async fn new(rpc_node_url: String) -> Self { + pub(crate) async fn new(rpc_url: String) -> Self { let headers = { let mut h = HeaderMap::new(); h.insert("Content-Type", HeaderValue::from_static("application/json")); @@ -66,7 +66,7 @@ impl RpcClient { }); let top_height = client - .get(&rpc_node_url) + .get(&rpc_url) .json(&request) .send() .await @@ -90,7 +90,7 @@ impl RpcClient { Self { client, - rpc_node_url, + rpc_url, top_height, } } @@ -103,7 +103,7 @@ impl RpcClient { "params": {"height": height, "fill_pow_hash": true} }); - tokio::task::spawn(self.client.get(&self.rpc_node_url).json(&request).send()) + tokio::task::spawn(self.client.get(&self.rpc_url).json(&request).send()) .await .unwrap() .unwrap() @@ -167,7 +167,7 @@ impl RpcClient { "\nheight: {height}\nheader: {header:#?}\nblock: {block:#?}" ); - let count = TESTED_BLOCK_COUNT.fetch_add(1, Ordering::Release); + let count = TESTED_BLOCK_COUNT.fetch_add(1, Ordering::Release) + 1; let hex_header = hex::encode(header.pow_hash); let hex_hash = hex::encode(pow_hash); From 7cd32544776a3c477f10c2b507dfd0ea63c60afa Mon Sep 17 00:00:00 2001 From: "hinto.janai" Date: Thu, 12 Dec 2024 17:56:57 -0500 Subject: [PATCH 04/16] clippy --- tests/monero-serai/src/main.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/monero-serai/src/main.rs b/tests/monero-serai/src/main.rs index 6bbf4701..2327b842 100644 --- a/tests/monero-serai/src/main.rs +++ b/tests/monero-serai/src/main.rs @@ -1,5 +1,4 @@ use std::{ - io::Write, sync::atomic::{AtomicUsize, Ordering}, time::{Duration, Instant}, }; From 8acb166639d16cec26c45ffc0a8956e0c2f80d48 Mon Sep 17 00:00:00 2001 From: "hinto.janai" Date: Thu, 12 Dec 2024 19:52:55 -0500 Subject: [PATCH 05/16] fixes --- Cargo.toml | 1 + tests/monero-serai/Cargo.toml | 2 +- tests/monero-serai/src/rpc.rs | 6 +++--- tests/pow/Cargo.toml | 2 +- tests/pow/src/rpc.rs | 12 ++++++------ 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index b9424371..f173551d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -163,6 +163,7 @@ pretty_assertions = { version = "1" } proptest = { version = "1" } proptest-derive = { version = "0.5" } tokio-test = { version = "0.4" } +reqwest = { version = "0.12" } ## TODO: ## Potential dependencies. diff --git a/tests/monero-serai/Cargo.toml b/tests/monero-serai/Cargo.toml index 8b246a78..d10f8c7d 100644 --- a/tests/monero-serai/Cargo.toml +++ b/tests/monero-serai/Cargo.toml @@ -9,7 +9,7 @@ hex = { workspace = true, features = ["serde", "std"] } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true, features = ["std"] } tokio = { workspace = true, features = ["full"] } -reqwest = { version = "0.12", features = ["json"] } +reqwest = { workspace = true, features = ["json"] } rayon = { workspace = true } futures = { workspace = true, features = ["std"] } diff --git a/tests/monero-serai/src/rpc.rs b/tests/monero-serai/src/rpc.rs index c08aa857..49e7dc26 100644 --- a/tests/monero-serai/src/rpc.rs +++ b/tests/monero-serai/src/rpc.rs @@ -256,12 +256,12 @@ impl RpcClient { assert_eq!(reward, block_reward, "{info}"); assert_eq!(timestamp, block.header.timestamp, "{info}"); - let block_count = TESTED_BLOCK_COUNT.fetch_add(1, Ordering::Release) + 1; + let progress = TESTED_BLOCK_COUNT.fetch_add(1, Ordering::Release) + 1; let tx_count = TESTED_TX_COUNT.fetch_add(num_txes, Ordering::Release) + 1; - let percent = (block_count as f64 / top_height as f64) * 100.0; + let percent = (progress as f64 / top_height as f64) * 100.0; println!( - "block_count | {block_count}/{top_height} ({percent:.2}%) + "progress | {progress}/{top_height} ({percent:.2}%) tx_count | {tx_count} hash | {} miner_tx_hash | {} diff --git a/tests/pow/Cargo.toml b/tests/pow/Cargo.toml index 4b965dd4..06a5743b 100644 --- a/tests/pow/Cargo.toml +++ b/tests/pow/Cargo.toml @@ -14,7 +14,7 @@ hex = { workspace = true, features = ["serde", "std"] } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true, features = ["std"] } tokio = { workspace = true, features = ["full"] } -reqwest = { version = "0.12", features = ["json"] } +reqwest = { workspace = true, features = ["json"] } rayon = { workspace = true } randomx-rs = { workspace = true } diff --git a/tests/pow/src/rpc.rs b/tests/pow/src/rpc.rs index 426c525e..8aea1957 100644 --- a/tests/pow/src/rpc.rs +++ b/tests/pow/src/rpc.rs @@ -66,7 +66,7 @@ impl RpcClient { }); let top_height = client - .get(&rpc_url) + .get(format!("{rpc_url}/json_rpc")) .json(&request) .send() .await @@ -103,7 +103,9 @@ impl RpcClient { "params": {"height": height, "fill_pow_hash": true} }); - tokio::task::spawn(self.client.get(&self.rpc_url).json(&request).send()) + let rpc_url = format!("{}/json_rpc", self.rpc_url); + + tokio::task::spawn(self.client.get(rpc_url).json(&request).send()) .await .unwrap() .unwrap() @@ -169,16 +171,14 @@ impl RpcClient { let count = TESTED_BLOCK_COUNT.fetch_add(1, Ordering::Release) + 1; - let hex_header = hex::encode(header.pow_hash); - let hex_hash = hex::encode(pow_hash); + let hash = hex::encode(pow_hash); let percent = (count as f64 / top_height as f64) * 100.0; println!( "progress | {count}/{top_height} ({percent:.2}%) height | {height} algo | {name} -header | {hex_header} -hash | {hex_hash}\n" +hash | {hash}\n" ); }); } From 065dd03ea7ba0a7e18021b1fc9634232fb85f157 Mon Sep 17 00:00:00 2001 From: "hinto.janai" Date: Thu, 12 Dec 2024 20:31:07 -0500 Subject: [PATCH 06/16] env vars --- Cargo.lock | 4 ++-- tests/monero-serai/Cargo.toml | 2 +- tests/monero-serai/src/main.rs | 33 +++++++++++++++++++++++++++------ tests/monero-serai/src/rpc.rs | 6 +++++- tests/pow/Cargo.toml | 2 +- tests/pow/src/main.rs | 25 +++++++++++++++++++++---- tests/pow/src/rpc.rs | 5 ++++- 7 files changed, 61 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3e96b6a1..40394be7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3319,7 +3319,7 @@ dependencies = [ [[package]] name = "tests-monero-serai" -version = "0.1.0" +version = "0.0.0" dependencies = [ "futures", "hex", @@ -3333,7 +3333,7 @@ dependencies = [ [[package]] name = "tests-pow" -version = "0.1.0" +version = "0.0.0" dependencies = [ "cuprate-consensus-rules", "cuprate-cryptonight", diff --git a/tests/monero-serai/Cargo.toml b/tests/monero-serai/Cargo.toml index d10f8c7d..0088e32f 100644 --- a/tests/monero-serai/Cargo.toml +++ b/tests/monero-serai/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tests-monero-serai" -version = "0.1.0" +version = "0.0.0" edition = "2021" [dependencies] diff --git a/tests/monero-serai/src/main.rs b/tests/monero-serai/src/main.rs index 2327b842..9dd8590d 100644 --- a/tests/monero-serai/src/main.rs +++ b/tests/monero-serai/src/main.rs @@ -13,14 +13,29 @@ async fn main() { let now = Instant::now(); let rpc_url = if let Ok(url) = std::env::var("RPC_URL") { + println!("RPC_URL (found): {url}"); url } else { - "http://127.0.0.1:18081".to_string() + let rpc_url = "http://127.0.0.1:18081".to_string(); + println!("RPC_URL (off, using default): {rpc_url}"); + rpc_url }; - println!("rpc_url: {rpc_url}"); + if std::env::var("VERBOSE").is_ok() { + println!("VERBOSE: true"); + } else { + println!("VERBOSE: false"); + } + + let mut client = rpc::RpcClient::new(rpc_url).await; - let client = rpc::RpcClient::new(rpc_url).await; - let top_height = client.top_height; + let top_height = if let Ok(Ok(h)) = std::env::var("TOP_HEIGHT").map(|s| s.parse()) { + client.top_height = h; + println!("TOP_HEIGHT (found): {h}"); + h + } else { + println!("TOP_HEIGHT (off, using latest): {}", client.top_height); + client.top_height + }; let ranges = (0..top_height) .collect::>() @@ -28,11 +43,17 @@ async fn main() { .map(<[usize]>::to_vec) .collect::>>(); - println!("ranges: "); + println!("ranges: ["); for range in &ranges { - println!("[{}..{}]", range.first().unwrap(), range.last().unwrap()); + println!( + " ({}..{}),", + range.first().unwrap(), + range.last().unwrap() + ); } + println!("]\n"); + let iter = ranges.into_iter().map(move |range| { let c = client.clone(); async move { diff --git a/tests/monero-serai/src/rpc.rs b/tests/monero-serai/src/rpc.rs index 49e7dc26..24559895 100644 --- a/tests/monero-serai/src/rpc.rs +++ b/tests/monero-serai/src/rpc.rs @@ -81,7 +81,6 @@ impl RpcClient { .block_header .height; - println!("top_height: {top_height}"); assert!(top_height > 3301441, "node is behind"); Self { @@ -258,6 +257,11 @@ impl RpcClient { let progress = TESTED_BLOCK_COUNT.fetch_add(1, Ordering::Release) + 1; let tx_count = TESTED_TX_COUNT.fetch_add(num_txes, Ordering::Release) + 1; + + if std::env::var("VERBOSE").is_err() && progress % 1000 != 0 { + return; + } + let percent = (progress as f64 / top_height as f64) * 100.0; println!( diff --git a/tests/pow/Cargo.toml b/tests/pow/Cargo.toml index 06a5743b..74a5f3a1 100644 --- a/tests/pow/Cargo.toml +++ b/tests/pow/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tests-pow" -version = "0.1.0" +version = "0.0.0" edition = "2021" [dependencies] diff --git a/tests/pow/src/main.rs b/tests/pow/src/main.rs index 9ac4d31e..22186a09 100644 --- a/tests/pow/src/main.rs +++ b/tests/pow/src/main.rs @@ -12,14 +12,31 @@ async fn main() { let now = Instant::now(); let rpc_url = if let Ok(url) = std::env::var("RPC_URL") { + println!("RPC_URL (found): {url}"); url } else { - "http://127.0.0.1:18081".to_string() + let rpc_url = "http://127.0.0.1:18081".to_string(); + println!("RPC_URL (off, using default): {rpc_url}"); + rpc_url + }; + if std::env::var("VERBOSE").is_ok() { + println!("VERBOSE: true"); + } else { + println!("VERBOSE: false"); + } + + let mut client = rpc::RpcClient::new(rpc_url).await; + + let top_height = if let Ok(Ok(h)) = std::env::var("TOP_HEIGHT").map(|s| s.parse()) { + client.top_height = h; + println!("TOP_HEIGHT (found): {h}"); + h + } else { + println!("TOP_HEIGHT (off, using latest): {}", client.top_height); + client.top_height }; - println!("rpc_url: {rpc_url}"); - let client = rpc::RpcClient::new(rpc_url).await; - let top_height = client.top_height; + println!(); tokio::join!( client.cryptonight_v0(), diff --git a/tests/pow/src/rpc.rs b/tests/pow/src/rpc.rs index 8aea1957..557d0efd 100644 --- a/tests/pow/src/rpc.rs +++ b/tests/pow/src/rpc.rs @@ -85,7 +85,6 @@ impl RpcClient { .try_into() .unwrap(); - println!("top_height: {top_height}"); assert!(top_height > 3301441, "node is behind"); Self { @@ -171,6 +170,10 @@ impl RpcClient { let count = TESTED_BLOCK_COUNT.fetch_add(1, Ordering::Release) + 1; + if std::env::var("VERBOSE").is_err() && count % 500 != 0 { + return; + } + let hash = hex::encode(pow_hash); let percent = (count as f64 / top_height as f64) * 100.0; From 6e682bdf39a8c1a5fd732d0861ca0487a7fe9c4b Mon Sep 17 00:00:00 2001 From: "hinto.janai" Date: Fri, 13 Dec 2024 08:53:02 -0500 Subject: [PATCH 07/16] handle 202612 pow hash --- Cargo.lock | 1 + tests/pow/Cargo.toml | 1 + tests/pow/src/rpc.rs | 29 +++++++++++++++++------------ 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 40394be7..be524f42 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3339,6 +3339,7 @@ dependencies = [ "cuprate-cryptonight", "function_name", "hex", + "hex-literal", "monero-serai", "randomx-rs", "rayon", diff --git a/tests/pow/Cargo.toml b/tests/pow/Cargo.toml index 74a5f3a1..bb17694b 100644 --- a/tests/pow/Cargo.toml +++ b/tests/pow/Cargo.toml @@ -11,6 +11,7 @@ function_name = { workspace = true } thread_local = { workspace = true } monero-serai = { workspace = true } hex = { workspace = true, features = ["serde", "std"] } +hex-literal = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true, features = ["std"] } tokio = { workspace = true, features = ["full"] } diff --git a/tests/pow/src/rpc.rs b/tests/pow/src/rpc.rs index 557d0efd..129cee6d 100644 --- a/tests/pow/src/rpc.rs +++ b/tests/pow/src/rpc.rs @@ -6,6 +6,7 @@ use std::{ use function_name::named; use hex::serde::deserialize; +use hex_literal::hex; use monero_serai::block::Block; use randomx_rs::{RandomXCache, RandomXFlag, RandomXVM}; use reqwest::{ @@ -114,7 +115,7 @@ impl RpcClient { .result } - async fn test( + async fn test( &self, range: Range, hash: impl Fn(Vec, u64, u64, [u8; 32]) -> [u8; 32] + Send + Sync + 'static + Copy, @@ -156,12 +157,16 @@ impl RpcClient { Err(e) => panic!("{e:?}\nblob: {blob:?}, header: {header:?}"), }; - let pow_hash = hash( - block.serialize_pow_hash(), - height.try_into().unwrap(), - seed_height.try_into().unwrap(), - seed_hash, - ); + let pow_hash = if CRYPTONIGHT_V0 && height == 202612 { + hex!("84f64766475d51837ac9efbef1926486e58563c95a19fef4aec3254f03000000") + } else { + hash( + block.serialize_pow_hash(), + height.try_into().unwrap(), + seed_height.try_into().unwrap(), + seed_hash, + ) + }; assert_eq!( header.pow_hash, pow_hash, @@ -189,7 +194,7 @@ hash | {hash}\n" #[named] pub(crate) async fn cryptonight_v0(&self) { - self.test::( + self.test::( 0..1546000, |b, _, _, _| cuprate_cryptonight::cryptonight_hash_v0(&b), function_name!(), @@ -199,7 +204,7 @@ hash | {hash}\n" #[named] pub(crate) async fn cryptonight_v1(&self) { - self.test::( + self.test::( 1546000..1685555, |b, _, _, _| cuprate_cryptonight::cryptonight_hash_v1(&b).unwrap(), function_name!(), @@ -209,7 +214,7 @@ hash | {hash}\n" #[named] pub(crate) async fn cryptonight_v2(&self) { - self.test::( + self.test::( 1685555..1788000, |b, _, _, _| cuprate_cryptonight::cryptonight_hash_v2(&b), function_name!(), @@ -219,7 +224,7 @@ hash | {hash}\n" #[named] pub(crate) async fn cryptonight_r(&self) { - self.test::( + self.test::( 1788000..1978433, |b, h, _, _| cuprate_cryptonight::cryptonight_hash_r(&b, h), function_name!(), @@ -251,7 +256,7 @@ hash | {hash}\n" .unwrap() }; - self.test::(1978433..self.top_height, function, function_name!()) + self.test::(1978433..self.top_height, function, function_name!()) .await; } } From 63378a0e961924ab09e3c432d0c77cc69c4aa14f Mon Sep 17 00:00:00 2001 From: "hinto.janai" Date: Fri, 13 Dec 2024 12:22:30 -0500 Subject: [PATCH 08/16] test all transactions are unique --- tests/monero-serai/src/rpc.rs | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/tests/monero-serai/src/rpc.rs b/tests/monero-serai/src/rpc.rs index 24559895..3bfeb61c 100644 --- a/tests/monero-serai/src/rpc.rs +++ b/tests/monero-serai/src/rpc.rs @@ -1,4 +1,7 @@ -use std::{collections::BTreeSet, sync::atomic::Ordering}; +use std::{ + collections::{BTreeSet, HashSet}, + sync::{atomic::Ordering, LazyLock}, +}; use hex::serde::deserialize; use monero_serai::{block::Block, transaction::Transaction}; @@ -9,6 +12,7 @@ use reqwest::{ }; use serde::Deserialize; use serde_json::json; +use tokio::sync::Mutex; use crate::{TESTED_BLOCK_COUNT, TESTED_TX_COUNT}; @@ -141,6 +145,7 @@ impl RpcClient { .collect() } + #[expect(clippy::significant_drop_tightening)] pub(crate) async fn get_block_test_batch(&self, heights: BTreeSet) { #[derive(Debug, Clone, Deserialize)] struct JsonRpcResponse { @@ -193,6 +198,23 @@ impl RpcClient { let txs = self.get_transactions(tx_hashes.clone()).await; assert_eq!(tx_hashes.len(), txs.len()); + // Test all transactions are unique. + { + static TX_SET: LazyLock>> = + LazyLock::new(|| Mutex::new(HashSet::new())); + + let tx_hashes = tx_hashes.clone(); + let mut tx_set = TX_SET.lock().await; + + for hash in tx_hashes { + assert!( + tx_set.insert(hash), + "duplicated tx hash: {}\n{info}", + hex::encode(hash), + ); + } + } + let top_height = self.top_height; #[expect(clippy::cast_precision_loss)] @@ -214,7 +236,6 @@ impl RpcClient { .sum::(); assert_ne!(block_reward, 0, "block reward is 0\n{info}"); - // Test fields are correct. let BlockHeader { block_weight, hash, @@ -242,6 +263,7 @@ impl RpcClient { assert!(matches!(tx.version(), 1 | 2), "{info}, tx: {tx:#?}"); }); + // Test block fields are correct. assert_eq!(block_weight, total_block_weight, "{info}"); assert_ne!(block.miner_transaction.weight(), 0, "{info}"); assert_eq!(hash, block.hash(), "{info}"); From 1e6f760d72a69de3e74dcb97c959164c82ca896c Mon Sep 17 00:00:00 2001 From: "hinto.janai" Date: Sun, 15 Dec 2024 21:09:19 -0500 Subject: [PATCH 09/16] pow: simplify concurrency pipeline --- Cargo.lock | 5 +- consensus/rules/src/blocks.rs | 4 +- tests/monero-serai/Cargo.toml | 1 + tests/monero-serai/src/rpc.rs | 42 +++++-- tests/pow/Cargo.toml | 23 ++-- tests/pow/src/cryptonight.rs | 62 +++++++++++ tests/pow/src/main.rs | 48 +++++--- tests/pow/src/randomx.rs | 43 ++++++++ tests/pow/src/rpc.rs | 202 ++++++++-------------------------- tests/pow/src/verify.rs | 105 ++++++++++++++++++ 10 files changed, 335 insertions(+), 200 deletions(-) create mode 100644 tests/pow/src/cryptonight.rs create mode 100644 tests/pow/src/randomx.rs create mode 100644 tests/pow/src/verify.rs diff --git a/Cargo.lock b/Cargo.lock index be524f42..ff298ebd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3335,18 +3335,17 @@ dependencies = [ name = "tests-pow" version = "0.0.0" dependencies = [ + "crossbeam", "cuprate-consensus-rules", "cuprate-cryptonight", - "function_name", + "futures", "hex", "hex-literal", "monero-serai", "randomx-rs", - "rayon", "reqwest", "serde", "serde_json", - "thread_local", "tokio", ] diff --git a/consensus/rules/src/blocks.rs b/consensus/rules/src/blocks.rs index 5e55ce2a..3ae64f6b 100644 --- a/consensus/rules/src/blocks.rs +++ b/consensus/rules/src/blocks.rs @@ -60,10 +60,10 @@ pub const fn is_randomx_seed_height(height: usize) -> bool { /// /// ref: pub const fn randomx_seed_height(height: usize) -> usize { - if height <= RX_SEEDHASH_EPOCH_BLOCKS + RX_SEEDHASH_EPOCH_LAG { + if height <= 2048 + 64 { 0 } else { - (height - RX_SEEDHASH_EPOCH_LAG - 1) & !(RX_SEEDHASH_EPOCH_BLOCKS - 1) + (height - 64 - 1) & !(2048 - 1) } } diff --git a/tests/monero-serai/Cargo.toml b/tests/monero-serai/Cargo.toml index 0088e32f..54c8a41a 100644 --- a/tests/monero-serai/Cargo.toml +++ b/tests/monero-serai/Cargo.toml @@ -11,6 +11,7 @@ serde_json = { workspace = true, features = ["std"] } tokio = { workspace = true, features = ["full"] } reqwest = { workspace = true, features = ["json"] } rayon = { workspace = true } +# rand = { workspace = true, features = ["std", "std_rng"] } futures = { workspace = true, features = ["std"] } [lints] diff --git a/tests/monero-serai/src/rpc.rs b/tests/monero-serai/src/rpc.rs index 3bfeb61c..ef213199 100644 --- a/tests/monero-serai/src/rpc.rs +++ b/tests/monero-serai/src/rpc.rs @@ -1,6 +1,7 @@ use std::{ collections::{BTreeSet, HashSet}, sync::{atomic::Ordering, LazyLock}, + time::Instant, }; use hex::serde::deserialize; @@ -39,6 +40,8 @@ pub(crate) struct BlockHeader { pub(crate) struct RpcClient { client: Client, rpc_url: String, + json_rpc_url: String, + get_transactions_url: String, pub top_height: usize, } @@ -72,8 +75,11 @@ impl RpcClient { "params": {} }); + let json_rpc_url = format!("{rpc_url}/json_rpc"); + let get_transactions_url = format!("{rpc_url}/get_transactions"); + let top_height = client - .get(format!("{rpc_url}/json_rpc")) + .get(&json_rpc_url) .json(&request) .send() .await @@ -90,6 +96,8 @@ impl RpcClient { Self { client, rpc_url, + json_rpc_url, + get_transactions_url, top_height, } } @@ -108,20 +116,16 @@ impl RpcClient { pub pruned_as_hex: String, } - let url = format!("{}/get_transactions", self.rpc_url); - let txs_hashes = tx_hashes .into_iter() .map(hex::encode) .collect::>(); - let request = json!({ - "txs_hashes": txs_hashes, - }); + let request = json!({"txs_hashes":txs_hashes}); let txs = self .client - .get(&url) + .get(&self.get_transactions_url) .json(&request) .send() .await @@ -145,7 +149,11 @@ impl RpcClient { .collect() } - #[expect(clippy::significant_drop_tightening)] + #[expect( + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + clippy::significant_drop_tightening + )] pub(crate) async fn get_block_test_batch(&self, heights: BTreeSet) { #[derive(Debug, Clone, Deserialize)] struct JsonRpcResponse { @@ -159,9 +167,9 @@ impl RpcClient { pub block_header: BlockHeader, } - let tasks = heights.into_iter().map(|height| { - let json_rpc_url = format!("{}/json_rpc", self.rpc_url); + let now = Instant::now(); + let tasks = heights.into_iter().map(|height| { let request = json!({ "jsonrpc": "2.0", "id": 0, @@ -169,7 +177,9 @@ impl RpcClient { "params": {"height": height} }); - let task = tokio::task::spawn(self.client.get(&json_rpc_url).json(&request).send()); + let task = + tokio::task::spawn(self.client.get(&self.json_rpc_url).json(&request).send()); + // tokio::task::spawn(self.client.get(&*self.nodes.rand()).json(&request).send()); (height, task) }); @@ -286,8 +296,16 @@ impl RpcClient { let percent = (progress as f64 / top_height as f64) * 100.0; + let elapsed = now.elapsed().as_secs_f64(); + let secs_per_hash = elapsed / progress as f64; + let bps = progress as f64 / elapsed; + let remaining_secs = (top_height as f64 - progress as f64) * secs_per_hash; + let h = (remaining_secs / 60.0 / 60.0) as u64; + let m = (remaining_secs / 60.0 % 60.0) as u64; + let s = (remaining_secs % 60.0) as u64; + println!( - "progress | {progress}/{top_height} ({percent:.2}%) + "progress | {progress}/{top_height} ({percent:.2}%, {bps:.2} blocks/sec, {h}h {m}m {s}s left) tx_count | {tx_count} hash | {} miner_tx_hash | {} diff --git a/tests/pow/Cargo.toml b/tests/pow/Cargo.toml index bb17694b..8544793e 100644 --- a/tests/pow/Cargo.toml +++ b/tests/pow/Cargo.toml @@ -4,20 +4,19 @@ version = "0.0.0" edition = "2021" [dependencies] -cuprate-cryptonight = { workspace = true } cuprate-consensus-rules = { workspace = true } +cuprate-cryptonight = { workspace = true } -function_name = { workspace = true } -thread_local = { workspace = true } -monero-serai = { workspace = true } -hex = { workspace = true, features = ["serde", "std"] } -hex-literal = { workspace = true } -serde = { workspace = true, features = ["derive"] } -serde_json = { workspace = true, features = ["std"] } -tokio = { workspace = true, features = ["full"] } -reqwest = { workspace = true, features = ["json"] } -rayon = { workspace = true } -randomx-rs = { workspace = true } +crossbeam = { workspace = true, features = ["std"] } +futures = { workspace = true, features = ["std"] } +monero-serai = { workspace = true } +hex = { workspace = true, features = ["serde", "std"] } +hex-literal = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true, features = ["std"] } +tokio = { workspace = true, features = ["full"] } +reqwest = { workspace = true, features = ["json"] } +randomx-rs = { workspace = true } [lints] workspace = true diff --git a/tests/pow/src/cryptonight.rs b/tests/pow/src/cryptonight.rs new file mode 100644 index 00000000..bab5e30c --- /dev/null +++ b/tests/pow/src/cryptonight.rs @@ -0,0 +1,62 @@ +use std::fmt::Display; + +use hex_literal::hex; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) enum CryptoNightHash { + V0, + V1, + V2, + R, +} + +impl CryptoNightHash { + /// The last height this hash function is used for proof-of-work. + pub(crate) const fn from_height(height: u64) -> Self { + if height < 1546000 { + Self::V0 + } else if height < 1685555 { + Self::V1 + } else if height < 1788000 { + Self::V2 + } else if height < 1978433 { + Self::R + } else { + panic!("height is large than 1978433"); + } + } + + pub(crate) fn hash(data: &[u8], height: u64) -> (&'static str, [u8; 32]) { + let this = Self::from_height(height); + + let hash = match Self::from_height(height) { + Self::V0 => { + if height == 202612 { + hex!("84f64766475d51837ac9efbef1926486e58563c95a19fef4aec3254f03000000") + } else { + cuprate_cryptonight::cryptonight_hash_v0(data) + } + } + Self::V1 => cuprate_cryptonight::cryptonight_hash_v1(data).unwrap(), + Self::V2 => cuprate_cryptonight::cryptonight_hash_v2(data), + Self::R => cuprate_cryptonight::cryptonight_hash_r(data, height), + }; + + (this.as_str(), hash) + } + + pub(crate) const fn as_str(self) -> &'static str { + match self { + Self::V0 => "cryptonight_v0", + Self::V1 => "cryptonight_v1", + Self::V2 => "cryptonight_v2", + Self::R => "cryptonight_r", + } + } +} + +impl Display for CryptoNightHash { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str((*self).as_str()) + } +} diff --git a/tests/pow/src/main.rs b/tests/pow/src/main.rs index 22186a09..149203f6 100644 --- a/tests/pow/src/main.rs +++ b/tests/pow/src/main.rs @@ -1,11 +1,25 @@ +mod cryptonight; +mod randomx; mod rpc; +mod verify; use std::{ - sync::atomic::{AtomicUsize, Ordering}, + sync::atomic::{AtomicU64, Ordering}, time::{Duration, Instant}, }; -pub static TESTED_BLOCK_COUNT: AtomicUsize = AtomicUsize::new(0); +use crate::rpc::GetBlockResponse; + +pub const RANDOMX_START_HEIGHT: u64 = 1978433; +pub static TESTED_BLOCK_COUNT: AtomicU64 = AtomicU64::new(0); + +#[derive(Debug)] +pub struct VerifyData { + pub get_block_response: GetBlockResponse, + pub height: u64, + pub seed_height: u64, + pub seed_hash: [u8; 32], +} #[tokio::main] async fn main() { @@ -25,32 +39,32 @@ async fn main() { println!("VERBOSE: false"); } - let mut client = rpc::RpcClient::new(rpc_url).await; + let client = rpc::RpcClient::new(rpc_url).await; + let top_height = client.top_height; + println!("top_height: {top_height}"); - let top_height = if let Ok(Ok(h)) = std::env::var("TOP_HEIGHT").map(|s| s.parse()) { - client.top_height = h; - println!("TOP_HEIGHT (found): {h}"); - h + let threads = if let Ok(Ok(c)) = std::env::var("THREADS").map(|s| s.parse()) { + println!("THREADS (found): {c}"); + c } else { - println!("TOP_HEIGHT (off, using latest): {}", client.top_height); - client.top_height + let c = std::thread::available_parallelism().unwrap().get(); + println!("THREADS (off): {c}"); + c }; println!(); - tokio::join!( - client.cryptonight_v0(), - client.cryptonight_v1(), - client.cryptonight_v2(), - client.cryptonight_r(), - client.randomx(), - ); + // Test RandomX. + let (tx, rx) = crossbeam::channel::unbounded(); + verify::spawn_verify_pool(threads, top_height, rx); + client.test(top_height, tx).await; + // Wait for other threads to finish. loop { let count = TESTED_BLOCK_COUNT.load(Ordering::Acquire); if top_height == count { - println!("finished all PoW, took {}s", now.elapsed().as_secs()); + println!("finished, took {}s", now.elapsed().as_secs()); std::process::exit(0); } diff --git a/tests/pow/src/randomx.rs b/tests/pow/src/randomx.rs new file mode 100644 index 00000000..d4d76a6a --- /dev/null +++ b/tests/pow/src/randomx.rs @@ -0,0 +1,43 @@ +use randomx_rs::{RandomXCache, RandomXDataset, RandomXFlag, RandomXVM}; + +/// Returns a [`RandomXVM`] with no optimization flags (default, light-verification). +pub(crate) fn randomx_vm_default(seed_hash: &[u8; 32]) -> RandomXVM { + const FLAG: RandomXFlag = RandomXFlag::FLAG_DEFAULT; + + let cache = RandomXCache::new(FLAG, seed_hash).unwrap(); + RandomXVM::new(FLAG, Some(cache), None).unwrap() +} + +/// Returns a [`RandomXVM`] with most optimization flags. +pub(crate) fn randomx_vm_optimized(seed_hash: &[u8; 32]) -> RandomXVM { + // TODO: conditional FLAG_LARGE_PAGES, FLAG_JIT + + let mut vm_flag = RandomXFlag::FLAG_FULL_MEM; + let mut cache_flag = RandomXFlag::empty(); + + #[cfg(target_arch = "x86_64")] + for flag in [&mut vm_flag, &mut cache_flag] { + if is_x86_feature_detected!("aes") { + *flag |= RandomXFlag::FLAG_HARD_AES; + } + + match ( + is_x86_feature_detected!("ssse3"), + is_x86_feature_detected!("avx2"), + ) { + (true, _) => *flag |= RandomXFlag::FLAG_ARGON2_SSSE3, + (_, true) => *flag |= RandomXFlag::FLAG_ARGON2_AVX2, + (_, _) => *flag |= RandomXFlag::FLAG_ARGON2, + } + } + + let hash = hex::encode(seed_hash); + + println!("Generating RandomX VM: seed_hash: {hash}, flags: {vm_flag:#?}"); + let cache = RandomXCache::new(cache_flag, seed_hash).unwrap(); + let dataset = RandomXDataset::new(RandomXFlag::FLAG_DEFAULT, cache, 0).unwrap(); + let vm = RandomXVM::new(vm_flag, None, Some(dataset)).unwrap(); + println!("Generating RandomX VM: seed_hash: {hash}, flags: {vm_flag:#?} ... OK"); + + vm +} diff --git a/tests/pow/src/rpc.rs b/tests/pow/src/rpc.rs index 129cee6d..e84fe58d 100644 --- a/tests/pow/src/rpc.rs +++ b/tests/pow/src/rpc.rs @@ -1,23 +1,15 @@ -use std::{ - collections::BTreeMap, - ops::Range, - sync::{atomic::Ordering, Mutex}, -}; +use std::time::Duration; -use function_name::named; +use crossbeam::channel::Sender; use hex::serde::deserialize; -use hex_literal::hex; -use monero_serai::block::Block; -use randomx_rs::{RandomXCache, RandomXFlag, RandomXVM}; use reqwest::{ header::{HeaderMap, HeaderValue}, Client, ClientBuilder, }; use serde::Deserialize; use serde_json::{json, Value}; -use thread_local::ThreadLocal; -use crate::TESTED_BLOCK_COUNT; +use crate::{VerifyData, RANDOMX_START_HEIGHT}; #[derive(Debug, Clone, Deserialize)] struct JsonRpcResponse { @@ -25,14 +17,14 @@ struct JsonRpcResponse { } #[derive(Debug, Clone, Deserialize)] -struct GetBlockResponse { +pub struct GetBlockResponse { #[serde(deserialize_with = "deserialize")] pub blob: Vec, pub block_header: BlockHeader, } #[derive(Debug, Clone, Deserialize)] -struct BlockHeader { +pub struct BlockHeader { #[serde(deserialize_with = "deserialize")] pub pow_hash: Vec, #[serde(deserialize_with = "deserialize")] @@ -42,8 +34,8 @@ struct BlockHeader { #[derive(Debug, Clone)] pub(crate) struct RpcClient { client: Client, - rpc_url: String, - pub top_height: usize, + json_rpc_url: String, + pub top_height: u64, } impl RpcClient { @@ -66,8 +58,10 @@ impl RpcClient { "params": {} }); + let json_rpc_url = format!("{rpc_url}/json_rpc"); + let top_height = client - .get(format!("{rpc_url}/json_rpc")) + .get(&json_rpc_url) .json(&request) .send() .await @@ -82,20 +76,18 @@ impl RpcClient { .get("height") .unwrap() .as_u64() - .unwrap() - .try_into() .unwrap(); assert!(top_height > 3301441, "node is behind"); Self { client, - rpc_url, + json_rpc_url, top_height, } } - async fn get_block(&self, height: usize) -> GetBlockResponse { + async fn get_block(&self, height: u64) -> GetBlockResponse { let request = json!({ "jsonrpc": "2.0", "id": 0, @@ -103,160 +95,62 @@ impl RpcClient { "params": {"height": height, "fill_pow_hash": true} }); - let rpc_url = format!("{}/json_rpc", self.rpc_url); - - tokio::task::spawn(self.client.get(rpc_url).json(&request).send()) + self.client + .get(&self.json_rpc_url) + .json(&request) + .send() .await .unwrap() - .unwrap() .json::() .await .unwrap() .result } - async fn test( - &self, - range: Range, - hash: impl Fn(Vec, u64, u64, [u8; 32]) -> [u8; 32] + Send + Sync + 'static + Copy, - name: &'static str, - ) { - let tasks = range.map(|height| { - let task = self.get_block(height); - (height, task) - }); + pub(crate) async fn test(self, top_height: u64, tx: Sender) { + use futures::StreamExt; - for (height, task) in tasks { - let result = task.await; + let iter = (0..top_height).map(|height| { + let this = &self; + let tx = tx.clone(); - let (seed_height, seed_hash) = if RANDOMX { - let seed_height = cuprate_consensus_rules::blocks::randomx_seed_height(height); + async move { + let get_block_response = this.get_block(height).await; - let seed_hash: [u8; 32] = self - .get_block(seed_height) - .await - .block_header - .hash + let (seed_height, seed_hash) = if height < RANDOMX_START_HEIGHT { + (0, [0; 32]) + } else { + let seed_height = cuprate_consensus_rules::blocks::randomx_seed_height( + height.try_into().unwrap(), + ) .try_into() .unwrap(); - (seed_height, seed_hash) - } else { - (0, [0; 32]) - }; - - let top_height = self.top_height; + let seed_hash = this + .get_block(seed_height) + .await + .block_header + .hash + .try_into() + .unwrap(); - #[expect(clippy::cast_precision_loss)] - rayon::spawn(move || { - let GetBlockResponse { blob, block_header } = result; - let header = block_header; - - let block = match Block::read(&mut blob.as_slice()) { - Ok(b) => b, - Err(e) => panic!("{e:?}\nblob: {blob:?}, header: {header:?}"), + (seed_height, seed_hash) }; - let pow_hash = if CRYPTONIGHT_V0 && height == 202612 { - hex!("84f64766475d51837ac9efbef1926486e58563c95a19fef4aec3254f03000000") - } else { - hash( - block.serialize_pow_hash(), - height.try_into().unwrap(), - seed_height.try_into().unwrap(), - seed_hash, - ) + let data = VerifyData { + get_block_response, + height, + seed_height, + seed_hash, }; - assert_eq!( - header.pow_hash, pow_hash, - "\nheight: {height}\nheader: {header:#?}\nblock: {block:#?}" - ); - - let count = TESTED_BLOCK_COUNT.fetch_add(1, Ordering::Release) + 1; - - if std::env::var("VERBOSE").is_err() && count % 500 != 0 { - return; - } - - let hash = hex::encode(pow_hash); - let percent = (count as f64 / top_height as f64) * 100.0; - - println!( - "progress | {count}/{top_height} ({percent:.2}%) -height | {height} -algo | {name} -hash | {hash}\n" - ); - }); - } - } - - #[named] - pub(crate) async fn cryptonight_v0(&self) { - self.test::( - 0..1546000, - |b, _, _, _| cuprate_cryptonight::cryptonight_hash_v0(&b), - function_name!(), - ) - .await; - } - - #[named] - pub(crate) async fn cryptonight_v1(&self) { - self.test::( - 1546000..1685555, - |b, _, _, _| cuprate_cryptonight::cryptonight_hash_v1(&b).unwrap(), - function_name!(), - ) - .await; - } - - #[named] - pub(crate) async fn cryptonight_v2(&self) { - self.test::( - 1685555..1788000, - |b, _, _, _| cuprate_cryptonight::cryptonight_hash_v2(&b), - function_name!(), - ) - .await; - } - - #[named] - pub(crate) async fn cryptonight_r(&self) { - self.test::( - 1788000..1978433, - |b, h, _, _| cuprate_cryptonight::cryptonight_hash_r(&b, h), - function_name!(), - ) - .await; - } - - #[named] - pub(crate) async fn randomx(&self) { - #[expect(clippy::significant_drop_tightening)] - let function = move |bytes: Vec, _, seed_height, seed_hash: [u8; 32]| { - static RANDOMX_VM: ThreadLocal>> = ThreadLocal::new(); - - let mut thread_local = RANDOMX_VM - .get_or(|| Mutex::new(BTreeMap::new())) - .lock() - .unwrap(); - - let randomx_vm = thread_local.entry(seed_height).or_insert_with(|| { - let flag = RandomXFlag::get_recommended_flags(); - let cache = RandomXCache::new(flag, &seed_hash).unwrap(); - RandomXVM::new(flag, Some(cache), None).unwrap() - }); - - randomx_vm - .calculate_hash(&bytes) - .unwrap() - .try_into() - .unwrap() - }; + tx.send(data).unwrap(); + } + }); - self.test::(1978433..self.top_height, function, function_name!()) + futures::stream::iter(iter) + .buffer_unordered(4) // This can't be too high or else we get bottlenecked by `monerod` + .for_each(|()| async {}) .await; } } diff --git a/tests/pow/src/verify.rs b/tests/pow/src/verify.rs new file mode 100644 index 00000000..0ed7fdd2 --- /dev/null +++ b/tests/pow/src/verify.rs @@ -0,0 +1,105 @@ +use std::{sync::atomic::Ordering, time::Instant}; + +use crossbeam::channel::Receiver; +use monero_serai::block::Block; + +use crate::{ + cryptonight::CryptoNightHash, rpc::GetBlockResponse, VerifyData, RANDOMX_START_HEIGHT, + TESTED_BLOCK_COUNT, +}; + +#[expect( + clippy::needless_pass_by_value, + clippy::cast_precision_loss, + clippy::cast_possible_truncation, + clippy::cast_sign_loss +)] +pub(crate) fn spawn_verify_pool(thread_count: usize, top_height: u64, rx: Receiver) { + let now = Instant::now(); + + for i in 0..thread_count { + let rx = rx.clone(); + + std::thread::spawn(move || { + let mut current_seed_hash = [0; 32]; + let mut randomx_vm = None; + + loop { + let Ok(data) = rx.recv() else { + println!("Exiting verify thread {i}/{thread_count}"); + return; + }; + + let VerifyData { + get_block_response, + height, + seed_height, + seed_hash, + } = data; + + let GetBlockResponse { blob, block_header } = get_block_response; + let header = block_header; + + let block = match Block::read(&mut blob.as_slice()) { + Ok(b) => b, + Err(e) => panic!("{e:?}\nblob: {blob:?}, header: {header:?}"), + }; + + let pow_data = block.serialize_pow_hash(); + + let (algo, pow_hash) = if height < RANDOMX_START_HEIGHT { + CryptoNightHash::hash(&pow_data, height) + } else { + if current_seed_hash != seed_hash { + randomx_vm = None; + } + + let randomx_vm = randomx_vm.get_or_insert_with(|| { + current_seed_hash = seed_hash; + // crate::randomx::randomx_vm_optimized(&seed_hash) + crate::randomx::randomx_vm_default(&seed_hash) + }); + + let pow_hash = randomx_vm + .calculate_hash(&pow_data) + .unwrap() + .try_into() + .unwrap(); + + ("randomx", pow_hash) + }; + + assert_eq!( + header.pow_hash, pow_hash, + "\nheight: {height}\nheader: {header:#?}\nblock: {block:#?}", + ); + + let count = TESTED_BLOCK_COUNT.fetch_add(1, Ordering::Release) + 1; + + if std::env::var("VERBOSE").is_err() && count % 500 != 0 { + continue; + } + + let pow_hash = hex::encode(pow_hash); + let seed_hash = hex::encode(seed_hash); + let percent = (count as f64 / top_height as f64) * 100.0; + + let elapsed = now.elapsed().as_secs_f64(); + let secs_per_hash = elapsed / count as f64; + let bps = count as f64 / elapsed; + let remaining_secs = (top_height as f64 - count as f64) * secs_per_hash; + let h = (remaining_secs / 60.0 / 60.0) as u64; + let m = (remaining_secs / 60.0 % 60.0) as u64; + let s = (remaining_secs % 60.0) as u64; + + println!( + "progress | {count}/{top_height} ({percent:.2}%, {bps:.2} blocks/sec, {h}h {m}m {s}s left) +algo | {algo} +seed_height | {seed_height} +seed_hash | {seed_hash} +pow_hash | {pow_hash}\n" + ); + } + }); + } +} From a6e506a8b6e15b937b7e919a7feb6cc1975896b2 Mon Sep 17 00:00:00 2001 From: "hinto.janai" Date: Sun, 15 Dec 2024 21:16:50 -0500 Subject: [PATCH 10/16] fixes --- consensus/rules/src/blocks.rs | 4 ++-- tests/monero-serai/Cargo.toml | 1 - tests/monero-serai/src/rpc.rs | 1 - 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/consensus/rules/src/blocks.rs b/consensus/rules/src/blocks.rs index 3ae64f6b..5e55ce2a 100644 --- a/consensus/rules/src/blocks.rs +++ b/consensus/rules/src/blocks.rs @@ -60,10 +60,10 @@ pub const fn is_randomx_seed_height(height: usize) -> bool { /// /// ref: pub const fn randomx_seed_height(height: usize) -> usize { - if height <= 2048 + 64 { + if height <= RX_SEEDHASH_EPOCH_BLOCKS + RX_SEEDHASH_EPOCH_LAG { 0 } else { - (height - 64 - 1) & !(2048 - 1) + (height - RX_SEEDHASH_EPOCH_LAG - 1) & !(RX_SEEDHASH_EPOCH_BLOCKS - 1) } } diff --git a/tests/monero-serai/Cargo.toml b/tests/monero-serai/Cargo.toml index 54c8a41a..0088e32f 100644 --- a/tests/monero-serai/Cargo.toml +++ b/tests/monero-serai/Cargo.toml @@ -11,7 +11,6 @@ serde_json = { workspace = true, features = ["std"] } tokio = { workspace = true, features = ["full"] } reqwest = { workspace = true, features = ["json"] } rayon = { workspace = true } -# rand = { workspace = true, features = ["std", "std_rng"] } futures = { workspace = true, features = ["std"] } [lints] diff --git a/tests/monero-serai/src/rpc.rs b/tests/monero-serai/src/rpc.rs index ef213199..62774bb8 100644 --- a/tests/monero-serai/src/rpc.rs +++ b/tests/monero-serai/src/rpc.rs @@ -179,7 +179,6 @@ impl RpcClient { let task = tokio::task::spawn(self.client.get(&self.json_rpc_url).json(&request).send()); - // tokio::task::spawn(self.client.get(&*self.nodes.rand()).json(&request).send()); (height, task) }); From a97367167b0ec392089708e208c449e9d1a7cda9 Mon Sep 17 00:00:00 2001 From: "hinto.janai" Date: Sun, 15 Dec 2024 23:12:52 -0500 Subject: [PATCH 11/16] combines tests into `compat` --- Cargo.lock | 87 +++++- Cargo.toml | 5 +- tests/{pow => compat}/Cargo.toml | 23 +- tests/compat/src/cli.rs | 30 ++ tests/compat/src/constants.rs | 10 + tests/{pow => compat}/src/cryptonight.rs | 8 +- tests/compat/src/main.rs | 53 ++++ tests/{pow => compat}/src/randomx.rs | 4 +- tests/{pow => compat}/src/rpc.rs | 115 ++++++-- tests/compat/src/types.rs | 69 +++++ tests/compat/src/verify.rs | 212 ++++++++++++++ tests/monero-serai/Cargo.toml | 17 -- tests/monero-serai/src/main.rs | 85 ------ tests/monero-serai/src/rpc.rs | 337 ----------------------- tests/pow/src/main.rs | 73 ----- tests/pow/src/verify.rs | 105 ------- 16 files changed, 558 insertions(+), 675 deletions(-) rename tests/{pow => compat}/Cargo.toml (50%) create mode 100644 tests/compat/src/cli.rs create mode 100644 tests/compat/src/constants.rs rename tests/{pow => compat}/src/cryptonight.rs (87%) create mode 100644 tests/compat/src/main.rs rename tests/{pow => compat}/src/randomx.rs (91%) rename tests/{pow => compat}/src/rpc.rs (52%) create mode 100644 tests/compat/src/types.rs create mode 100644 tests/compat/src/verify.rs delete mode 100644 tests/monero-serai/Cargo.toml delete mode 100644 tests/monero-serai/src/main.rs delete mode 100644 tests/monero-serai/src/rpc.rs delete mode 100644 tests/pow/src/main.rs delete mode 100644 tests/pow/src/verify.rs diff --git a/Cargo.lock b/Cargo.lock index ff298ebd..6efecf08 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -59,12 +59,55 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + [[package]] name = "anstyle" version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" +dependencies = [ + "anstyle", + "windows-sys 0.59.0", +] + [[package]] name = "anyhow" version = "1.0.92" @@ -444,8 +487,10 @@ version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54" dependencies = [ + "anstream", "anstyle", "clap_lex", + "strsim", "terminal_size", ] @@ -467,6 +512,12 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + [[package]] name = "const_format" version = "0.2.33" @@ -1898,6 +1949,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itertools" version = "0.10.5" @@ -3192,6 +3249,12 @@ dependencies = [ "spin", ] +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "strum" version = "0.26.3" @@ -3318,23 +3381,10 @@ dependencies = [ ] [[package]] -name = "tests-monero-serai" -version = "0.0.0" -dependencies = [ - "futures", - "hex", - "monero-serai", - "rayon", - "reqwest", - "serde", - "serde_json", - "tokio", -] - -[[package]] -name = "tests-pow" +name = "tests-compat" version = "0.0.0" dependencies = [ + "clap", "crossbeam", "cuprate-consensus-rules", "cuprate-cryptonight", @@ -3343,6 +3393,7 @@ dependencies = [ "hex-literal", "monero-serai", "randomx-rs", + "rayon", "reqwest", "serde", "serde_json", @@ -3730,6 +3781,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "valuable" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index f173551d..65ea523e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,12 +54,11 @@ members = [ "types", # Tests - "tests/pow", - "tests/monero-serai", + "tests/compat", ] [profile.release] -panic = "abort" +panic = "abort" lto = true # Build with LTO strip = "none" # Keep panic stack traces codegen-units = 1 # Optimize for binary speed over compile times diff --git a/tests/pow/Cargo.toml b/tests/compat/Cargo.toml similarity index 50% rename from tests/pow/Cargo.toml rename to tests/compat/Cargo.toml index 8544793e..15d8bc89 100644 --- a/tests/pow/Cargo.toml +++ b/tests/compat/Cargo.toml @@ -1,15 +1,23 @@ [package] -name = "tests-pow" -version = "0.0.0" -edition = "2021" +name = "tests-compat" +version = "0.0.0" +edition = "2021" +description = "Compatability tests between `cuprated` and `monerod`" +license = "MIT" +authors = ["hinto-janai"] +repository = "https://github.com/Cuprate/cuprate/tree/main/tests/compat" +keywords = ["cuprate", "tests", "compat"] + [dependencies] cuprate-consensus-rules = { workspace = true } -cuprate-cryptonight = { workspace = true } +cuprate-cryptonight = { workspace = true } +clap = { workspace = true, features = ["cargo", "derive", "default"] } crossbeam = { workspace = true, features = ["std"] } futures = { workspace = true, features = ["std"] } monero-serai = { workspace = true } +rayon = { workspace = true } hex = { workspace = true, features = ["serde", "std"] } hex-literal = { workspace = true } serde = { workspace = true, features = ["derive"] } @@ -20,3 +28,10 @@ randomx-rs = { workspace = true } [lints] workspace = true + +[profile.release] +panic = "unwind" +lto = true +strip = "none" +codegen-units = 1 +opt-level = 3 \ No newline at end of file diff --git a/tests/compat/src/cli.rs b/tests/compat/src/cli.rs new file mode 100644 index 00000000..5cd89c12 --- /dev/null +++ b/tests/compat/src/cli.rs @@ -0,0 +1,30 @@ +use std::num::{NonZeroU64, NonZeroUsize}; + +use clap::Parser; + +/// `cuprate` <-> `monerod` compatability tester. +#[derive(Parser, Debug)] +#[command(about, long_about = None)] +pub struct Args { + /// Name of the person to greet + #[arg(short, long, default_value_t = String::from("http://127.0.0.1:18081"))] + pub rpc_url: String, + + /// Amount of verifying threads to spawn. + #[arg(short, long, default_value_t = std::thread::available_parallelism().unwrap())] + pub threads: NonZeroUsize, + + /// Print an update every `update` amount of blocks. + #[arg(short, long, default_value_t = NonZeroU64::new(500).unwrap())] + pub update: NonZeroU64, +} + +impl Args { + pub fn get() -> Self { + let this = Self::parse(); + + println!("{this:#?}"); + + this + } +} diff --git a/tests/compat/src/constants.rs b/tests/compat/src/constants.rs new file mode 100644 index 00000000..157c2651 --- /dev/null +++ b/tests/compat/src/constants.rs @@ -0,0 +1,10 @@ +use std::sync::atomic::{AtomicU64, AtomicUsize}; + +/// Height at which RandomX activated. +pub const RANDOMX_START_HEIGHT: u64 = 1978433; + +/// Total amount of blocks tested, used as a global counter. +pub static TESTED_BLOCK_COUNT: AtomicU64 = AtomicU64::new(0); + +/// Total amount of transactions tested, used as a global counter. +pub static TESTED_TX_COUNT: AtomicUsize = AtomicUsize::new(0); diff --git a/tests/pow/src/cryptonight.rs b/tests/compat/src/cryptonight.rs similarity index 87% rename from tests/pow/src/cryptonight.rs rename to tests/compat/src/cryptonight.rs index bab5e30c..9711a494 100644 --- a/tests/pow/src/cryptonight.rs +++ b/tests/compat/src/cryptonight.rs @@ -3,7 +3,7 @@ use std::fmt::Display; use hex_literal::hex; #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub(crate) enum CryptoNightHash { +pub enum CryptoNightHash { V0, V1, V2, @@ -12,7 +12,7 @@ pub(crate) enum CryptoNightHash { impl CryptoNightHash { /// The last height this hash function is used for proof-of-work. - pub(crate) const fn from_height(height: u64) -> Self { + pub const fn from_height(height: u64) -> Self { if height < 1546000 { Self::V0 } else if height < 1685555 { @@ -26,7 +26,7 @@ impl CryptoNightHash { } } - pub(crate) fn hash(data: &[u8], height: u64) -> (&'static str, [u8; 32]) { + pub fn hash(data: &[u8], height: u64) -> (&'static str, [u8; 32]) { let this = Self::from_height(height); let hash = match Self::from_height(height) { @@ -45,7 +45,7 @@ impl CryptoNightHash { (this.as_str(), hash) } - pub(crate) const fn as_str(self) -> &'static str { + pub const fn as_str(self) -> &'static str { match self { Self::V0 => "cryptonight_v0", Self::V1 => "cryptonight_v1", diff --git a/tests/compat/src/main.rs b/tests/compat/src/main.rs new file mode 100644 index 00000000..1b8034e0 --- /dev/null +++ b/tests/compat/src/main.rs @@ -0,0 +1,53 @@ +#![allow( + clippy::doc_markdown, + reason = "TODO: add exception to doc clippy for `RandomX`" +)] +#![allow(unreachable_pub, reason = "This is a binary, everything `pub` is ok")] + +mod cli; +mod constants; +mod cryptonight; +mod randomx; +mod rpc; +mod types; +mod verify; + +use std::{ + sync::atomic::Ordering, + time::{Duration, Instant}, +}; + +#[tokio::main] +async fn main() { + let now = Instant::now(); + + // Parse CLI args. + let cli::Args { + rpc_url, + update, + threads, + } = cli::Args::get(); + + // Set-up RPC client. + let client = rpc::RpcClient::new(rpc_url).await; + let top_height = client.top_height; + println!("top_height: {top_height}"); + println!(); + + // Test. + let (tx, rx) = crossbeam::channel::unbounded(); + verify::spawn_verify_pool(threads, update, top_height, rx); + client.test(top_height, tx).await; + + // Wait for other threads to finish. + loop { + let count = constants::TESTED_BLOCK_COUNT.load(Ordering::Acquire); + + if top_height == count { + println!("finished, took {}s", now.elapsed().as_secs()); + std::process::exit(0); + } + + std::thread::sleep(Duration::from_secs(1)); + } +} diff --git a/tests/pow/src/randomx.rs b/tests/compat/src/randomx.rs similarity index 91% rename from tests/pow/src/randomx.rs rename to tests/compat/src/randomx.rs index d4d76a6a..7fcb92f8 100644 --- a/tests/pow/src/randomx.rs +++ b/tests/compat/src/randomx.rs @@ -1,7 +1,7 @@ use randomx_rs::{RandomXCache, RandomXDataset, RandomXFlag, RandomXVM}; /// Returns a [`RandomXVM`] with no optimization flags (default, light-verification). -pub(crate) fn randomx_vm_default(seed_hash: &[u8; 32]) -> RandomXVM { +pub fn randomx_vm_default(seed_hash: &[u8; 32]) -> RandomXVM { const FLAG: RandomXFlag = RandomXFlag::FLAG_DEFAULT; let cache = RandomXCache::new(FLAG, seed_hash).unwrap(); @@ -9,7 +9,7 @@ pub(crate) fn randomx_vm_default(seed_hash: &[u8; 32]) -> RandomXVM { } /// Returns a [`RandomXVM`] with most optimization flags. -pub(crate) fn randomx_vm_optimized(seed_hash: &[u8; 32]) -> RandomXVM { +pub fn randomx_vm_optimized(seed_hash: &[u8; 32]) -> RandomXVM { // TODO: conditional FLAG_LARGE_PAGES, FLAG_JIT let mut vm_flag = RandomXFlag::FLAG_FULL_MEM; diff --git a/tests/pow/src/rpc.rs b/tests/compat/src/rpc.rs similarity index 52% rename from tests/pow/src/rpc.rs rename to tests/compat/src/rpc.rs index e84fe58d..189e4d89 100644 --- a/tests/pow/src/rpc.rs +++ b/tests/compat/src/rpc.rs @@ -1,7 +1,6 @@ -use std::time::Duration; - use crossbeam::channel::Sender; -use hex::serde::deserialize; +use monero_serai::{block::Block, transaction::Transaction}; +use rayon::iter::{IndexedParallelIterator, IntoParallelIterator, ParallelIterator}; use reqwest::{ header::{HeaderMap, HeaderValue}, Client, ClientBuilder, @@ -9,37 +8,21 @@ use reqwest::{ use serde::Deserialize; use serde_json::{json, Value}; -use crate::{VerifyData, RANDOMX_START_HEIGHT}; - -#[derive(Debug, Clone, Deserialize)] -struct JsonRpcResponse { - result: GetBlockResponse, -} - -#[derive(Debug, Clone, Deserialize)] -pub struct GetBlockResponse { - #[serde(deserialize_with = "deserialize")] - pub blob: Vec, - pub block_header: BlockHeader, -} - -#[derive(Debug, Clone, Deserialize)] -pub struct BlockHeader { - #[serde(deserialize_with = "deserialize")] - pub pow_hash: Vec, - #[serde(deserialize_with = "deserialize")] - pub hash: Vec, -} +use crate::{ + constants::RANDOMX_START_HEIGHT, + types::{GetBlockResponse, JsonRpcResponse, RpcBlockData, RpcTxData}, +}; #[derive(Debug, Clone)] -pub(crate) struct RpcClient { +pub struct RpcClient { client: Client, json_rpc_url: String, + get_transactions_url: String, pub top_height: u64, } impl RpcClient { - pub(crate) async fn new(rpc_url: String) -> Self { + pub async fn new(rpc_url: String) -> Self { let headers = { let mut h = HeaderMap::new(); h.insert("Content-Type", HeaderValue::from_static("application/json")); @@ -59,6 +42,7 @@ impl RpcClient { }); let json_rpc_url = format!("{rpc_url}/json_rpc"); + let get_transactions_url = format!("{rpc_url}/get_transactions"); let top_height = client .get(&json_rpc_url) @@ -83,6 +67,7 @@ impl RpcClient { Self { client, json_rpc_url, + get_transactions_url, top_height, } } @@ -107,16 +92,85 @@ impl RpcClient { .result } - pub(crate) async fn test(self, top_height: u64, tx: Sender) { + async fn get_transactions(&self, tx_hashes: Vec<[u8; 32]>) -> Vec { + assert!(!tx_hashes.is_empty()); + + #[derive(Debug, Clone, Deserialize)] + struct GetTransactionsResponse { + txs: Vec, + } + + #[derive(Debug, Clone, Deserialize)] + struct Tx { + as_hex: String, + pruned_as_hex: String, + } + + let txs_hashes = tx_hashes.iter().map(hex::encode).collect::>(); + let request = json!({"txs_hashes":txs_hashes}); + + let txs = self + .client + .get(&self.get_transactions_url) + .json(&request) + .send() + .await + .unwrap() + .json::() + .await + .unwrap() + .txs; + + assert_eq!(txs.len(), tx_hashes.len()); + + txs.into_par_iter() + .zip(tx_hashes) + .map(|(r, tx_hash)| { + let tx_blob = hex::decode(if r.as_hex.is_empty() { + r.pruned_as_hex + } else { + r.as_hex + }) + .unwrap(); + + let tx = Transaction::read(&mut tx_blob.as_slice()).unwrap(); + + RpcTxData { + tx, + tx_blob, + tx_hash, + } + }) + .collect() + } + + pub async fn test(self, top_height: u64, tx: Sender) { use futures::StreamExt; let iter = (0..top_height).map(|height| { - let this = &self; + let this = self.clone(); let tx = tx.clone(); async move { let get_block_response = this.get_block(height).await; + let (this, get_block_response, block, txs) = + tokio::task::spawn_blocking(move || async move { + // Deserialize the block. + let block = Block::read(&mut get_block_response.blob.as_slice()).unwrap(); + + // Fetch and deserialize all transactions. + let mut tx_hashes = Vec::with_capacity(block.transactions.len() + 1); + tx_hashes.push(block.miner_transaction.hash()); + tx_hashes.extend(block.transactions.iter()); + let txs = this.get_transactions(tx_hashes).await; + + (this, get_block_response, block, txs) + }) + .await + .unwrap() + .await; + let (seed_height, seed_hash) = if height < RANDOMX_START_HEIGHT { (0, [0; 32]) } else { @@ -137,11 +191,12 @@ impl RpcClient { (seed_height, seed_hash) }; - let data = VerifyData { + let data = RpcBlockData { get_block_response, - height, + block, seed_height, seed_hash, + txs, }; tx.send(data).unwrap(); diff --git a/tests/compat/src/types.rs b/tests/compat/src/types.rs new file mode 100644 index 00000000..5277b506 --- /dev/null +++ b/tests/compat/src/types.rs @@ -0,0 +1,69 @@ +use hex::serde::deserialize; +use monero_serai::{block::Block, transaction::Transaction}; +use serde::Deserialize; + +/// Data of a single block from RPC. +#[derive(Debug)] +pub struct RpcBlockData { + /// Subset of JSON-RPC `get_block` data. + pub get_block_response: GetBlockResponse, + + /// The block itself. + pub block: Block, + + /// The correct seed height needed for this block for `RandomX`. + pub seed_height: u64, + /// The correct seed hash needed for this block for `RandomX`. + pub seed_hash: [u8; 32], + + /// All transactions in the block. + /// This vec is: + /// - the original transaction blobs + pub txs: Vec, +} + +/// Data of a transaction. +#[derive(Debug)] +pub struct RpcTxData { + /// The transactions itself. + pub tx: Transaction, + /// The transactions blob. + pub tx_blob: Vec, + /// The transaction's hash. + pub tx_hash: [u8; 32], +} + +/// Subset of JSON-RPC `get_block` response. +#[derive(Debug, Clone, Deserialize)] +pub struct JsonRpcResponse { + pub result: GetBlockResponse, +} + +/// Subset of JSON-RPC `get_block` data. +#[derive(Debug, Clone, Deserialize)] +pub struct GetBlockResponse { + #[serde(deserialize_with = "deserialize")] + pub blob: Vec, + pub block_header: BlockHeader, +} + +#[derive(Debug, Clone, Deserialize)] +pub(crate) struct BlockHeader { + #[serde(deserialize_with = "deserialize")] + pub hash: [u8; 32], + #[serde(deserialize_with = "deserialize")] + pub pow_hash: [u8; 32], + #[serde(deserialize_with = "deserialize")] + pub miner_tx_hash: [u8; 32], + #[serde(deserialize_with = "deserialize")] + pub prev_hash: [u8; 32], + + pub block_weight: usize, + pub height: u64, + pub major_version: u8, + pub minor_version: u8, + pub nonce: u32, + pub num_txes: usize, + pub reward: u64, + pub timestamp: u64, +} diff --git a/tests/compat/src/verify.rs b/tests/compat/src/verify.rs new file mode 100644 index 00000000..b482ebf7 --- /dev/null +++ b/tests/compat/src/verify.rs @@ -0,0 +1,212 @@ +use std::{ + collections::HashSet, + num::{NonZeroU64, NonZeroUsize}, + sync::{atomic::Ordering, LazyLock, Mutex}, + time::Instant, +}; + +use crossbeam::channel::Receiver; + +use crate::{ + constants::{RANDOMX_START_HEIGHT, TESTED_BLOCK_COUNT, TESTED_TX_COUNT}, + cryptonight::CryptoNightHash, + types::{BlockHeader, GetBlockResponse, RpcBlockData, RpcTxData}, +}; + +#[expect( + clippy::needless_pass_by_value, + clippy::cast_precision_loss, + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + clippy::significant_drop_tightening +)] +pub fn spawn_verify_pool( + thread_count: NonZeroUsize, + update: NonZeroU64, + top_height: u64, + rx: Receiver, +) { + let now = Instant::now(); + + for i in 0..thread_count.get() { + let rx = rx.clone(); + + std::thread::spawn(move || { + let mut current_seed_hash = [0; 32]; + let mut randomx_vm = None; + + loop { + let Ok(data) = rx.recv() else { + println!("Exiting verify thread {i}/{thread_count}"); + return; + }; + + // Panic info. + let p = format!("data: {data:#?}"); + + let RpcBlockData { + get_block_response, + block, + seed_height, + seed_hash, + txs, + } = data; + let GetBlockResponse { blob, block_header } = get_block_response; + let BlockHeader { + block_weight, + hash, + pow_hash, + height, + major_version, + minor_version, + miner_tx_hash, + nonce, + num_txes, + prev_hash, + reward, + timestamp, + } = block_header; + + // Test block properties. + assert_eq!(blob, block.serialize(), "{p:#?}"); + + assert!( + !block.miner_transaction.prefix().outputs.is_empty(), + "miner_tx has no outputs\n{p:#?}" + ); + + let block_reward = block + .miner_transaction + .prefix() + .outputs + .iter() + .map(|o| o.amount.unwrap()) + .sum::(); + assert_ne!(block_reward, 0, "block reward is 0\n{p:#?}"); + + let total_block_weight = txs + .iter() + .map(|RpcTxData { tx, .. }| tx.weight()) + .sum::(); + + // Test all transactions are unique. + { + static TX_SET: LazyLock>> = + LazyLock::new(|| Mutex::new(HashSet::new())); + + let mut tx_set = TX_SET.lock().unwrap(); + + for tx_hash in txs.iter().map(|RpcTxData { tx_hash, .. }| tx_hash) { + assert!( + tx_set.insert(*tx_hash), + "duplicated tx_hash: {}, {p:#?}", + hex::encode(tx_hash), + ); + } + } + + // Test transaction properties. + for RpcTxData { + tx, + tx_blob, + tx_hash, + } in txs + { + assert_eq!(tx_hash, tx.hash(), "{p:#?}, tx: {tx:#?}"); + assert_ne!(tx.weight(), 0, "{p:#?}, tx: {tx:#?}"); + assert!(!tx.prefix().inputs.is_empty(), "{p:#?}, tx: {tx:#?}"); + assert_eq!(tx_blob, tx.serialize(), "{p:#?}, tx: {tx:#?}"); + assert!(matches!(tx.version(), 1 | 2), "{p:#?}, tx: {tx:#?}"); + } + + // Test block fields are correct. + assert_eq!(block_weight, total_block_weight, "{p:#?}"); + assert_ne!(block.miner_transaction.weight(), 0, "{p:#?}"); + assert_eq!(hash, block.hash(), "{p:#?}"); + assert_eq!( + height, + u64::try_from(block.number().unwrap()).unwrap(), + "{p:#?}" + ); + assert_eq!(major_version, block.header.hardfork_version, "{p:#?}"); + assert_eq!(minor_version, block.header.hardfork_signal, "{p:#?}"); + assert_eq!(miner_tx_hash, block.miner_transaction.hash(), "{p:#?}"); + assert_eq!(nonce, block.header.nonce, "{p:#?}"); + assert_eq!(num_txes, block.transactions.len(), "{p:#?}"); + assert_eq!(prev_hash, block.header.previous, "{p:#?}"); + assert_eq!(reward, block_reward, "{p:#?}"); + assert_eq!(timestamp, block.header.timestamp, "{p:#?}"); + + // + let pow_data = block.serialize_pow_hash(); + + let (algo, calculated_pow_hash) = if height < RANDOMX_START_HEIGHT { + CryptoNightHash::hash(&pow_data, height) + } else { + if current_seed_hash != seed_hash { + randomx_vm = None; + } + + let randomx_vm = randomx_vm.get_or_insert_with(|| { + current_seed_hash = seed_hash; + // crate::randomx::randomx_vm_optimized(&seed_hash) + crate::randomx::randomx_vm_default(&seed_hash) + }); + + let pow_hash = randomx_vm + .calculate_hash(&pow_data) + .unwrap() + .try_into() + .unwrap(); + + ("randomx", pow_hash) + }; + + assert_eq!(calculated_pow_hash, pow_hash, "{p:#?}",); + + let count = TESTED_BLOCK_COUNT.fetch_add(1, Ordering::Release) + 1; + let total_tx_count = TESTED_TX_COUNT.fetch_add(num_txes, Ordering::Release) + 1; + + if count % update.get() != 0 { + continue; + } + + let pow_hash = hex::encode(pow_hash); + let seed_hash = hex::encode(seed_hash); + let percent = (count as f64 / top_height as f64) * 100.0; + + let elapsed = now.elapsed().as_secs_f64(); + let secs_per_hash = elapsed / count as f64; + let bps = count as f64 / elapsed; + let remaining_secs = (top_height as f64 - count as f64) * secs_per_hash; + let h = (remaining_secs / 60.0 / 60.0) as u64; + let m = (remaining_secs / 60.0 % 60.0) as u64; + let s = (remaining_secs % 60.0) as u64; + + let block_hash = hex::encode(hash); + let miner_tx_hash = hex::encode(miner_tx_hash); + let prev_hash = hex::encode(prev_hash); + let miner_tx_weight = block.miner_transaction.weight(); + + println!("progress | {count}/{top_height} ({percent:.2}%, {algo}, {bps:.2} blocks/sec, {h}h {m}m {s}s left) +seed_hash | {seed_hash} +pow_hash | {pow_hash} +block_hash | {block_hash} +miner_tx_hash | {miner_tx_hash} +prev_hash | {prev_hash} +reward | {reward} +timestamp | {timestamp} +nonce | {nonce} +total_tx_count | {total_tx_count} +height | {height} +seed_height | {seed_height} +block_weight | {block_weight} +miner_tx_weight | {miner_tx_weight} +major_version | {major_version} +minor_version | {minor_version} +num_txes | {num_txes}\n", + ); + } + }); + } +} diff --git a/tests/monero-serai/Cargo.toml b/tests/monero-serai/Cargo.toml deleted file mode 100644 index 0088e32f..00000000 --- a/tests/monero-serai/Cargo.toml +++ /dev/null @@ -1,17 +0,0 @@ -[package] -name = "tests-monero-serai" -version = "0.0.0" -edition = "2021" - -[dependencies] -monero-serai = { workspace = true } -hex = { workspace = true, features = ["serde", "std"] } -serde = { workspace = true, features = ["derive"] } -serde_json = { workspace = true, features = ["std"] } -tokio = { workspace = true, features = ["full"] } -reqwest = { workspace = true, features = ["json"] } -rayon = { workspace = true } -futures = { workspace = true, features = ["std"] } - -[lints] -workspace = true diff --git a/tests/monero-serai/src/main.rs b/tests/monero-serai/src/main.rs deleted file mode 100644 index 9dd8590d..00000000 --- a/tests/monero-serai/src/main.rs +++ /dev/null @@ -1,85 +0,0 @@ -use std::{ - sync::atomic::{AtomicUsize, Ordering}, - time::{Duration, Instant}, -}; - -mod rpc; - -pub static TESTED_BLOCK_COUNT: AtomicUsize = AtomicUsize::new(0); -pub static TESTED_TX_COUNT: AtomicUsize = AtomicUsize::new(0); - -#[tokio::main] -async fn main() { - let now = Instant::now(); - - let rpc_url = if let Ok(url) = std::env::var("RPC_URL") { - println!("RPC_URL (found): {url}"); - url - } else { - let rpc_url = "http://127.0.0.1:18081".to_string(); - println!("RPC_URL (off, using default): {rpc_url}"); - rpc_url - }; - if std::env::var("VERBOSE").is_ok() { - println!("VERBOSE: true"); - } else { - println!("VERBOSE: false"); - } - - let mut client = rpc::RpcClient::new(rpc_url).await; - - let top_height = if let Ok(Ok(h)) = std::env::var("TOP_HEIGHT").map(|s| s.parse()) { - client.top_height = h; - println!("TOP_HEIGHT (found): {h}"); - h - } else { - println!("TOP_HEIGHT (off, using latest): {}", client.top_height); - client.top_height - }; - - let ranges = (0..top_height) - .collect::>() - .chunks(100_000) - .map(<[usize]>::to_vec) - .collect::>>(); - - println!("ranges: ["); - for range in &ranges { - println!( - " ({}..{}),", - range.first().unwrap(), - range.last().unwrap() - ); - } - - println!("]\n"); - - let iter = ranges.into_iter().map(move |range| { - let c = client.clone(); - async move { - tokio::task::spawn_blocking(move || async move { - c.get_block_test_batch(range.into_iter().collect()).await; - }) - .await - .unwrap() - .await; - } - }); - - futures::future::join_all(iter).await; - - loop { - let block_count = TESTED_BLOCK_COUNT.load(Ordering::Acquire); - let tx_count = TESTED_TX_COUNT.load(Ordering::Acquire); - - if top_height == block_count { - println!( - "finished processing: blocks: {block_count}/{top_height}, txs: {tx_count}, took {}s", - now.elapsed().as_secs() - ); - std::process::exit(0); - } - - std::thread::sleep(Duration::from_secs(1)); - } -} diff --git a/tests/monero-serai/src/rpc.rs b/tests/monero-serai/src/rpc.rs deleted file mode 100644 index 62774bb8..00000000 --- a/tests/monero-serai/src/rpc.rs +++ /dev/null @@ -1,337 +0,0 @@ -use std::{ - collections::{BTreeSet, HashSet}, - sync::{atomic::Ordering, LazyLock}, - time::Instant, -}; - -use hex::serde::deserialize; -use monero_serai::{block::Block, transaction::Transaction}; -use rayon::iter::{IndexedParallelIterator, IntoParallelIterator, ParallelIterator}; -use reqwest::{ - header::{HeaderMap, HeaderValue}, - Client, ClientBuilder, -}; -use serde::Deserialize; -use serde_json::json; -use tokio::sync::Mutex; - -use crate::{TESTED_BLOCK_COUNT, TESTED_TX_COUNT}; - -#[derive(Debug, Clone, Deserialize)] -pub(crate) struct BlockHeader { - #[serde(deserialize_with = "deserialize")] - pub hash: Vec, - #[serde(deserialize_with = "deserialize")] - pub miner_tx_hash: Vec, - #[serde(deserialize_with = "deserialize")] - pub prev_hash: Vec, - - pub block_weight: usize, - pub height: usize, - pub major_version: u8, - pub minor_version: u8, - pub nonce: u32, - pub num_txes: usize, - pub reward: u64, - pub timestamp: u64, -} - -#[derive(Debug, Clone)] -pub(crate) struct RpcClient { - client: Client, - rpc_url: String, - json_rpc_url: String, - get_transactions_url: String, - pub top_height: usize, -} - -impl RpcClient { - pub(crate) async fn new(rpc_url: String) -> Self { - let headers = { - let mut h = HeaderMap::new(); - h.insert("Content-Type", HeaderValue::from_static("application/json")); - h - }; - - let client = ClientBuilder::new() - .default_headers(headers) - .build() - .unwrap(); - - #[derive(Debug, Clone, Deserialize)] - struct JsonRpcResponse { - result: GetLastBlockHeaderResponse, - } - - #[derive(Debug, Clone, Deserialize)] - pub(crate) struct GetLastBlockHeaderResponse { - pub block_header: BlockHeader, - } - - let request = json!({ - "jsonrpc": "2.0", - "id": 0, - "method": "get_last_block_header", - "params": {} - }); - - let json_rpc_url = format!("{rpc_url}/json_rpc"); - let get_transactions_url = format!("{rpc_url}/get_transactions"); - - let top_height = client - .get(&json_rpc_url) - .json(&request) - .send() - .await - .unwrap() - .json::() - .await - .unwrap() - .result - .block_header - .height; - - assert!(top_height > 3301441, "node is behind"); - - Self { - client, - rpc_url, - json_rpc_url, - get_transactions_url, - top_height, - } - } - - async fn get_transactions(&self, tx_hashes: Vec<[u8; 32]>) -> Vec<(Transaction, Vec)> { - assert!(!tx_hashes.is_empty()); - - #[derive(Debug, Clone, Deserialize)] - pub(crate) struct GetTransactionsResponse { - pub txs: Vec, - } - - #[derive(Debug, Clone, Deserialize)] - pub(crate) struct Tx { - pub as_hex: String, - pub pruned_as_hex: String, - } - - let txs_hashes = tx_hashes - .into_iter() - .map(hex::encode) - .collect::>(); - - let request = json!({"txs_hashes":txs_hashes}); - - let txs = self - .client - .get(&self.get_transactions_url) - .json(&request) - .send() - .await - .unwrap() - .json::() - .await - .unwrap() - .txs; - - txs.into_par_iter() - .map(|r| { - let blob = hex::decode(if r.as_hex.is_empty() { - r.pruned_as_hex - } else { - r.as_hex - }) - .unwrap(); - - (Transaction::read(&mut blob.as_slice()).unwrap(), blob) - }) - .collect() - } - - #[expect( - clippy::cast_possible_truncation, - clippy::cast_sign_loss, - clippy::significant_drop_tightening - )] - pub(crate) async fn get_block_test_batch(&self, heights: BTreeSet) { - #[derive(Debug, Clone, Deserialize)] - struct JsonRpcResponse { - result: GetBlockResponse, - } - - #[derive(Debug, Clone, Deserialize)] - pub(crate) struct GetBlockResponse { - #[serde(deserialize_with = "deserialize")] - pub blob: Vec, - pub block_header: BlockHeader, - } - - let now = Instant::now(); - - let tasks = heights.into_iter().map(|height| { - let request = json!({ - "jsonrpc": "2.0", - "id": 0, - "method": "get_block", - "params": {"height": height} - }); - - let task = - tokio::task::spawn(self.client.get(&self.json_rpc_url).json(&request).send()); - - (height, task) - }); - - for (height, task) in tasks { - let resp = task - .await - .unwrap() - .unwrap() - .json::() - .await - .unwrap() - .result; - - let info = format!("\nheight: {height}\nresponse: {resp:#?}"); - - // Test block deserialization. - let block = match Block::read(&mut resp.blob.as_slice()) { - Ok(b) => b, - Err(e) => panic!("{e:?}\n{info}"), - }; - - // Fetch all transactions. - let mut tx_hashes = vec![block.miner_transaction.hash()]; - tx_hashes.extend(block.transactions.iter()); - let txs = self.get_transactions(tx_hashes.clone()).await; - assert_eq!(tx_hashes.len(), txs.len()); - - // Test all transactions are unique. - { - static TX_SET: LazyLock>> = - LazyLock::new(|| Mutex::new(HashSet::new())); - - let tx_hashes = tx_hashes.clone(); - let mut tx_set = TX_SET.lock().await; - - for hash in tx_hashes { - assert!( - tx_set.insert(hash), - "duplicated tx hash: {}\n{info}", - hex::encode(hash), - ); - } - } - - let top_height = self.top_height; - - #[expect(clippy::cast_precision_loss)] - rayon::spawn(move || { - // Test block properties. - assert_eq!(resp.blob, block.serialize(), "{info}"); - - assert!( - !block.miner_transaction.prefix().outputs.is_empty(), - "miner_tx has no outputs\n{info}" - ); - - let block_reward = block - .miner_transaction - .prefix() - .outputs - .iter() - .map(|o| o.amount.unwrap()) - .sum::(); - assert_ne!(block_reward, 0, "block reward is 0\n{info}"); - - let BlockHeader { - block_weight, - hash, - height, - major_version, - minor_version, - miner_tx_hash, - nonce, - num_txes, - prev_hash, - reward, - timestamp, - } = resp.block_header; - - let total_block_weight = txs.iter().map(|(tx, _)| tx.weight()).sum::(); - - // Test transaction properties. - txs.into_par_iter() - .zip(tx_hashes) - .for_each(|((tx, blob), hash)| { - assert_eq!(hash, tx.hash(), "{info}, tx: {tx:#?}"); - assert_ne!(tx.weight(), 0, "{info}, tx: {tx:#?}"); - assert!(!tx.prefix().inputs.is_empty(), "{info}, tx: {tx:#?}"); - assert_eq!(blob, tx.serialize(), "{info}, tx: {tx:#?}"); - assert!(matches!(tx.version(), 1 | 2), "{info}, tx: {tx:#?}"); - }); - - // Test block fields are correct. - assert_eq!(block_weight, total_block_weight, "{info}"); - assert_ne!(block.miner_transaction.weight(), 0, "{info}"); - assert_eq!(hash, block.hash(), "{info}"); - assert_eq!(height, block.number().unwrap(), "{info}"); - assert_eq!(major_version, block.header.hardfork_version, "{info}"); - assert_eq!(minor_version, block.header.hardfork_signal, "{info}"); - assert_eq!(miner_tx_hash, block.miner_transaction.hash(), "{info}"); - assert_eq!(nonce, block.header.nonce, "{info}"); - assert_eq!(num_txes, block.transactions.len(), "{info}"); - assert_eq!(prev_hash, block.header.previous, "{info}"); - assert_eq!(reward, block_reward, "{info}"); - assert_eq!(timestamp, block.header.timestamp, "{info}"); - - let progress = TESTED_BLOCK_COUNT.fetch_add(1, Ordering::Release) + 1; - let tx_count = TESTED_TX_COUNT.fetch_add(num_txes, Ordering::Release) + 1; - - if std::env::var("VERBOSE").is_err() && progress % 1000 != 0 { - return; - } - - let percent = (progress as f64 / top_height as f64) * 100.0; - - let elapsed = now.elapsed().as_secs_f64(); - let secs_per_hash = elapsed / progress as f64; - let bps = progress as f64 / elapsed; - let remaining_secs = (top_height as f64 - progress as f64) * secs_per_hash; - let h = (remaining_secs / 60.0 / 60.0) as u64; - let m = (remaining_secs / 60.0 % 60.0) as u64; - let s = (remaining_secs % 60.0) as u64; - - println!( - "progress | {progress}/{top_height} ({percent:.2}%, {bps:.2} blocks/sec, {h}h {m}m {s}s left) -tx_count | {tx_count} -hash | {} -miner_tx_hash | {} -prev_hash | {} -reward | {} -timestamp | {} -nonce | {} -height | {} -block_weight | {} -miner_tx_weight | {} -major_version | {} -minor_version | {} -num_txes | {}\n", - hex::encode(hash), - hex::encode(miner_tx_hash), - hex::encode(prev_hash), - reward, - timestamp, - nonce, - height, - block_weight, - block.miner_transaction.weight(), - major_version, - minor_version, - num_txes, - ); - }); - } - } -} diff --git a/tests/pow/src/main.rs b/tests/pow/src/main.rs deleted file mode 100644 index 149203f6..00000000 --- a/tests/pow/src/main.rs +++ /dev/null @@ -1,73 +0,0 @@ -mod cryptonight; -mod randomx; -mod rpc; -mod verify; - -use std::{ - sync::atomic::{AtomicU64, Ordering}, - time::{Duration, Instant}, -}; - -use crate::rpc::GetBlockResponse; - -pub const RANDOMX_START_HEIGHT: u64 = 1978433; -pub static TESTED_BLOCK_COUNT: AtomicU64 = AtomicU64::new(0); - -#[derive(Debug)] -pub struct VerifyData { - pub get_block_response: GetBlockResponse, - pub height: u64, - pub seed_height: u64, - pub seed_hash: [u8; 32], -} - -#[tokio::main] -async fn main() { - let now = Instant::now(); - - let rpc_url = if let Ok(url) = std::env::var("RPC_URL") { - println!("RPC_URL (found): {url}"); - url - } else { - let rpc_url = "http://127.0.0.1:18081".to_string(); - println!("RPC_URL (off, using default): {rpc_url}"); - rpc_url - }; - if std::env::var("VERBOSE").is_ok() { - println!("VERBOSE: true"); - } else { - println!("VERBOSE: false"); - } - - let client = rpc::RpcClient::new(rpc_url).await; - let top_height = client.top_height; - println!("top_height: {top_height}"); - - let threads = if let Ok(Ok(c)) = std::env::var("THREADS").map(|s| s.parse()) { - println!("THREADS (found): {c}"); - c - } else { - let c = std::thread::available_parallelism().unwrap().get(); - println!("THREADS (off): {c}"); - c - }; - - println!(); - - // Test RandomX. - let (tx, rx) = crossbeam::channel::unbounded(); - verify::spawn_verify_pool(threads, top_height, rx); - client.test(top_height, tx).await; - - // Wait for other threads to finish. - loop { - let count = TESTED_BLOCK_COUNT.load(Ordering::Acquire); - - if top_height == count { - println!("finished, took {}s", now.elapsed().as_secs()); - std::process::exit(0); - } - - std::thread::sleep(Duration::from_secs(1)); - } -} diff --git a/tests/pow/src/verify.rs b/tests/pow/src/verify.rs deleted file mode 100644 index 0ed7fdd2..00000000 --- a/tests/pow/src/verify.rs +++ /dev/null @@ -1,105 +0,0 @@ -use std::{sync::atomic::Ordering, time::Instant}; - -use crossbeam::channel::Receiver; -use monero_serai::block::Block; - -use crate::{ - cryptonight::CryptoNightHash, rpc::GetBlockResponse, VerifyData, RANDOMX_START_HEIGHT, - TESTED_BLOCK_COUNT, -}; - -#[expect( - clippy::needless_pass_by_value, - clippy::cast_precision_loss, - clippy::cast_possible_truncation, - clippy::cast_sign_loss -)] -pub(crate) fn spawn_verify_pool(thread_count: usize, top_height: u64, rx: Receiver) { - let now = Instant::now(); - - for i in 0..thread_count { - let rx = rx.clone(); - - std::thread::spawn(move || { - let mut current_seed_hash = [0; 32]; - let mut randomx_vm = None; - - loop { - let Ok(data) = rx.recv() else { - println!("Exiting verify thread {i}/{thread_count}"); - return; - }; - - let VerifyData { - get_block_response, - height, - seed_height, - seed_hash, - } = data; - - let GetBlockResponse { blob, block_header } = get_block_response; - let header = block_header; - - let block = match Block::read(&mut blob.as_slice()) { - Ok(b) => b, - Err(e) => panic!("{e:?}\nblob: {blob:?}, header: {header:?}"), - }; - - let pow_data = block.serialize_pow_hash(); - - let (algo, pow_hash) = if height < RANDOMX_START_HEIGHT { - CryptoNightHash::hash(&pow_data, height) - } else { - if current_seed_hash != seed_hash { - randomx_vm = None; - } - - let randomx_vm = randomx_vm.get_or_insert_with(|| { - current_seed_hash = seed_hash; - // crate::randomx::randomx_vm_optimized(&seed_hash) - crate::randomx::randomx_vm_default(&seed_hash) - }); - - let pow_hash = randomx_vm - .calculate_hash(&pow_data) - .unwrap() - .try_into() - .unwrap(); - - ("randomx", pow_hash) - }; - - assert_eq!( - header.pow_hash, pow_hash, - "\nheight: {height}\nheader: {header:#?}\nblock: {block:#?}", - ); - - let count = TESTED_BLOCK_COUNT.fetch_add(1, Ordering::Release) + 1; - - if std::env::var("VERBOSE").is_err() && count % 500 != 0 { - continue; - } - - let pow_hash = hex::encode(pow_hash); - let seed_hash = hex::encode(seed_hash); - let percent = (count as f64 / top_height as f64) * 100.0; - - let elapsed = now.elapsed().as_secs_f64(); - let secs_per_hash = elapsed / count as f64; - let bps = count as f64 / elapsed; - let remaining_secs = (top_height as f64 - count as f64) * secs_per_hash; - let h = (remaining_secs / 60.0 / 60.0) as u64; - let m = (remaining_secs / 60.0 % 60.0) as u64; - let s = (remaining_secs % 60.0) as u64; - - println!( - "progress | {count}/{top_height} ({percent:.2}%, {bps:.2} blocks/sec, {h}h {m}m {s}s left) -algo | {algo} -seed_height | {seed_height} -seed_hash | {seed_hash} -pow_hash | {pow_hash}\n" - ); - } - }); - } -} From 6f855319666548320398559194677c0a312718c7 Mon Sep 17 00:00:00 2001 From: "hinto.janai" Date: Sun, 15 Dec 2024 23:31:37 -0500 Subject: [PATCH 12/16] lints --- Cargo.lock | 41 ++++++++++++++++++++-------------------- clippy.toml | 6 ++++++ tests/compat/Cargo.toml | 14 ++++---------- tests/compat/src/cli.rs | 12 ++++++++++-- tests/compat/src/main.rs | 4 ---- 5 files changed, 41 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6efecf08..2727e509 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1150,6 +1150,27 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "cuprate-tests-compat" +version = "0.0.0" +dependencies = [ + "clap", + "crossbeam", + "cuprate-consensus-rules", + "cuprate-constants", + "cuprate-cryptonight", + "futures", + "hex", + "hex-literal", + "monero-serai", + "randomx-rs", + "rayon", + "reqwest", + "serde", + "serde_json", + "tokio", +] + [[package]] name = "cuprate-txpool" version = "0.0.0" @@ -3380,26 +3401,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "tests-compat" -version = "0.0.0" -dependencies = [ - "clap", - "crossbeam", - "cuprate-consensus-rules", - "cuprate-cryptonight", - "futures", - "hex", - "hex-literal", - "monero-serai", - "randomx-rs", - "rayon", - "reqwest", - "serde", - "serde_json", - "tokio", -] - [[package]] name = "thiserror" version = "1.0.66" diff --git a/clippy.toml b/clippy.toml index cc94ec53..ec060525 100644 --- a/clippy.toml +++ b/clippy.toml @@ -1 +1,7 @@ +# upper-case-acronyms-aggressive = true + +# +doc-valid-idents = [ + "RandomX", ".." +] diff --git a/tests/compat/Cargo.toml b/tests/compat/Cargo.toml index 15d8bc89..9678cfb4 100644 --- a/tests/compat/Cargo.toml +++ b/tests/compat/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "tests-compat" +name = "cuprate-tests-compat" version = "0.0.0" edition = "2021" description = "Compatability tests between `cuprated` and `monerod`" @@ -10,10 +10,11 @@ keywords = ["cuprate", "tests", "compat"] [dependencies] +cuprate-constants = { workspace = true, features = ["build",] } cuprate-consensus-rules = { workspace = true } cuprate-cryptonight = { workspace = true } -clap = { workspace = true, features = ["cargo", "derive", "default"] } +clap = { workspace = true, features = ["cargo", "derive", "default", "string"] } crossbeam = { workspace = true, features = ["std"] } futures = { workspace = true, features = ["std"] } monero-serai = { workspace = true } @@ -27,11 +28,4 @@ reqwest = { workspace = true, features = ["json"] } randomx-rs = { workspace = true } [lints] -workspace = true - -[profile.release] -panic = "unwind" -lto = true -strip = "none" -codegen-units = 1 -opt-level = 3 \ No newline at end of file +workspace = true \ No newline at end of file diff --git a/tests/compat/src/cli.rs b/tests/compat/src/cli.rs index 5cd89c12..ed9dda95 100644 --- a/tests/compat/src/cli.rs +++ b/tests/compat/src/cli.rs @@ -2,9 +2,17 @@ use std::num::{NonZeroU64, NonZeroUsize}; use clap::Parser; -/// `cuprate` <-> `monerod` compatability tester. +/// `cuprate` <-> `monerod` compatibility tester. #[derive(Parser, Debug)] -#[command(about, long_about = None)] +#[command( + about, + long_about = None, + long_version = format!( + "{} {}", + clap::crate_version!(), + cuprate_constants::build::COMMIT + ), +)] pub struct Args { /// Name of the person to greet #[arg(short, long, default_value_t = String::from("http://127.0.0.1:18081"))] diff --git a/tests/compat/src/main.rs b/tests/compat/src/main.rs index 1b8034e0..21767a05 100644 --- a/tests/compat/src/main.rs +++ b/tests/compat/src/main.rs @@ -1,7 +1,3 @@ -#![allow( - clippy::doc_markdown, - reason = "TODO: add exception to doc clippy for `RandomX`" -)] #![allow(unreachable_pub, reason = "This is a binary, everything `pub` is ok")] mod cli; From e6c96a69fc7ad35e1ee02dae62172d26ffa8ecec Mon Sep 17 00:00:00 2001 From: "hinto.janai" Date: Mon, 16 Dec 2024 00:38:54 -0500 Subject: [PATCH 13/16] verify: split into methods --- tests/compat/src/main.rs | 2 +- tests/compat/src/randomx.rs | 21 +- tests/compat/src/rpc.rs | 8 +- tests/compat/src/types.rs | 2 +- tests/compat/src/verify.rs | 444 ++++++++++++++++++++++-------------- 5 files changed, 283 insertions(+), 194 deletions(-) diff --git a/tests/compat/src/main.rs b/tests/compat/src/main.rs index 21767a05..c6923e57 100644 --- a/tests/compat/src/main.rs +++ b/tests/compat/src/main.rs @@ -40,7 +40,7 @@ async fn main() { let count = constants::TESTED_BLOCK_COUNT.load(Ordering::Acquire); if top_height == count { - println!("finished, took {}s", now.elapsed().as_secs()); + println!("Finished, took {}s", now.elapsed().as_secs()); std::process::exit(0); } diff --git a/tests/compat/src/randomx.rs b/tests/compat/src/randomx.rs index 7fcb92f8..6e3d8b4c 100644 --- a/tests/compat/src/randomx.rs +++ b/tests/compat/src/randomx.rs @@ -9,27 +9,12 @@ pub fn randomx_vm_default(seed_hash: &[u8; 32]) -> RandomXVM { } /// Returns a [`RandomXVM`] with most optimization flags. +#[expect(dead_code)] pub fn randomx_vm_optimized(seed_hash: &[u8; 32]) -> RandomXVM { // TODO: conditional FLAG_LARGE_PAGES, FLAG_JIT - let mut vm_flag = RandomXFlag::FLAG_FULL_MEM; - let mut cache_flag = RandomXFlag::empty(); - - #[cfg(target_arch = "x86_64")] - for flag in [&mut vm_flag, &mut cache_flag] { - if is_x86_feature_detected!("aes") { - *flag |= RandomXFlag::FLAG_HARD_AES; - } - - match ( - is_x86_feature_detected!("ssse3"), - is_x86_feature_detected!("avx2"), - ) { - (true, _) => *flag |= RandomXFlag::FLAG_ARGON2_SSSE3, - (_, true) => *flag |= RandomXFlag::FLAG_ARGON2_AVX2, - (_, _) => *flag |= RandomXFlag::FLAG_ARGON2, - } - } + let vm_flag = RandomXFlag::get_recommended_flags() | RandomXFlag::FLAG_FULL_MEM; + let cache_flag = RandomXFlag::get_recommended_flags(); let hash = hex::encode(seed_hash); diff --git a/tests/compat/src/rpc.rs b/tests/compat/src/rpc.rs index 189e4d89..33153171 100644 --- a/tests/compat/src/rpc.rs +++ b/tests/compat/src/rpc.rs @@ -180,13 +180,7 @@ impl RpcClient { .try_into() .unwrap(); - let seed_hash = this - .get_block(seed_height) - .await - .block_header - .hash - .try_into() - .unwrap(); + let seed_hash = this.get_block(seed_height).await.block_header.hash; (seed_height, seed_hash) }; diff --git a/tests/compat/src/types.rs b/tests/compat/src/types.rs index 5277b506..1c9aac27 100644 --- a/tests/compat/src/types.rs +++ b/tests/compat/src/types.rs @@ -47,7 +47,7 @@ pub struct GetBlockResponse { pub block_header: BlockHeader, } -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Copy, Clone, Deserialize)] pub(crate) struct BlockHeader { #[serde(deserialize_with = "deserialize")] pub hash: [u8; 32], diff --git a/tests/compat/src/verify.rs b/tests/compat/src/verify.rs index b482ebf7..309fb667 100644 --- a/tests/compat/src/verify.rs +++ b/tests/compat/src/verify.rs @@ -6,6 +6,8 @@ use std::{ }; use crossbeam::channel::Receiver; +use monero_serai::block::Block; +use randomx_rs::RandomXVM; use crate::{ constants::{RANDOMX_START_HEIGHT, TESTED_BLOCK_COUNT, TESTED_TX_COUNT}, @@ -13,13 +15,19 @@ use crate::{ types::{BlockHeader, GetBlockResponse, RpcBlockData, RpcTxData}, }; -#[expect( - clippy::needless_pass_by_value, - clippy::cast_precision_loss, - clippy::cast_possible_truncation, - clippy::cast_sign_loss, - clippy::significant_drop_tightening -)] +struct Verifier { + id: usize, + now: Instant, + thread_count: NonZeroUsize, + update: NonZeroU64, + top_height: u64, + rx: Receiver, + seed_hash: [u8; 32], + timestamp: u64, + randomx_vm: Option, +} + +#[expect(clippy::needless_pass_by_value)] pub fn spawn_verify_pool( thread_count: NonZeroUsize, update: NonZeroU64, @@ -28,167 +36,271 @@ pub fn spawn_verify_pool( ) { let now = Instant::now(); - for i in 0..thread_count.get() { + for id in 0..thread_count.get() { let rx = rx.clone(); - std::thread::spawn(move || { - let mut current_seed_hash = [0; 32]; - let mut randomx_vm = None; - - loop { - let Ok(data) = rx.recv() else { - println!("Exiting verify thread {i}/{thread_count}"); - return; - }; - - // Panic info. - let p = format!("data: {data:#?}"); - - let RpcBlockData { - get_block_response, - block, - seed_height, - seed_hash, - txs, - } = data; - let GetBlockResponse { blob, block_header } = get_block_response; - let BlockHeader { - block_weight, - hash, - pow_hash, - height, - major_version, - minor_version, - miner_tx_hash, - nonce, - num_txes, - prev_hash, - reward, - timestamp, - } = block_header; - - // Test block properties. - assert_eq!(blob, block.serialize(), "{p:#?}"); - - assert!( - !block.miner_transaction.prefix().outputs.is_empty(), - "miner_tx has no outputs\n{p:#?}" - ); + Verifier { + id, + now, + thread_count, + update, + top_height, + rx, + seed_hash: [0; 32], + timestamp: 0, + randomx_vm: None, + } + .loop_listen_verify(); + }); + } +} - let block_reward = block - .miner_transaction - .prefix() - .outputs - .iter() - .map(|o| o.amount.unwrap()) - .sum::(); - assert_ne!(block_reward, 0, "block reward is 0\n{p:#?}"); - - let total_block_weight = txs - .iter() - .map(|RpcTxData { tx, .. }| tx.weight()) - .sum::(); - - // Test all transactions are unique. - { - static TX_SET: LazyLock>> = - LazyLock::new(|| Mutex::new(HashSet::new())); - - let mut tx_set = TX_SET.lock().unwrap(); - - for tx_hash in txs.iter().map(|RpcTxData { tx_hash, .. }| tx_hash) { - assert!( - tx_set.insert(*tx_hash), - "duplicated tx_hash: {}, {p:#?}", - hex::encode(tx_hash), - ); - } - } - - // Test transaction properties. - for RpcTxData { - tx, - tx_blob, - tx_hash, - } in txs - { - assert_eq!(tx_hash, tx.hash(), "{p:#?}, tx: {tx:#?}"); - assert_ne!(tx.weight(), 0, "{p:#?}, tx: {tx:#?}"); - assert!(!tx.prefix().inputs.is_empty(), "{p:#?}, tx: {tx:#?}"); - assert_eq!(tx_blob, tx.serialize(), "{p:#?}, tx: {tx:#?}"); - assert!(matches!(tx.version(), 1 | 2), "{p:#?}, tx: {tx:#?}"); - } - - // Test block fields are correct. - assert_eq!(block_weight, total_block_weight, "{p:#?}"); - assert_ne!(block.miner_transaction.weight(), 0, "{p:#?}"); - assert_eq!(hash, block.hash(), "{p:#?}"); - assert_eq!( - height, - u64::try_from(block.number().unwrap()).unwrap(), - "{p:#?}" - ); - assert_eq!(major_version, block.header.hardfork_version, "{p:#?}"); - assert_eq!(minor_version, block.header.hardfork_signal, "{p:#?}"); - assert_eq!(miner_tx_hash, block.miner_transaction.hash(), "{p:#?}"); - assert_eq!(nonce, block.header.nonce, "{p:#?}"); - assert_eq!(num_txes, block.transactions.len(), "{p:#?}"); - assert_eq!(prev_hash, block.header.previous, "{p:#?}"); - assert_eq!(reward, block_reward, "{p:#?}"); - assert_eq!(timestamp, block.header.timestamp, "{p:#?}"); - - // - let pow_data = block.serialize_pow_hash(); - - let (algo, calculated_pow_hash) = if height < RANDOMX_START_HEIGHT { - CryptoNightHash::hash(&pow_data, height) - } else { - if current_seed_hash != seed_hash { - randomx_vm = None; - } - - let randomx_vm = randomx_vm.get_or_insert_with(|| { - current_seed_hash = seed_hash; - // crate::randomx::randomx_vm_optimized(&seed_hash) - crate::randomx::randomx_vm_default(&seed_hash) - }); - - let pow_hash = randomx_vm - .calculate_hash(&pow_data) - .unwrap() - .try_into() - .unwrap(); - - ("randomx", pow_hash) - }; - - assert_eq!(calculated_pow_hash, pow_hash, "{p:#?}",); - - let count = TESTED_BLOCK_COUNT.fetch_add(1, Ordering::Release) + 1; - let total_tx_count = TESTED_TX_COUNT.fetch_add(num_txes, Ordering::Release) + 1; - - if count % update.get() != 0 { - continue; - } - - let pow_hash = hex::encode(pow_hash); - let seed_hash = hex::encode(seed_hash); - let percent = (count as f64 / top_height as f64) * 100.0; - - let elapsed = now.elapsed().as_secs_f64(); - let secs_per_hash = elapsed / count as f64; - let bps = count as f64 / elapsed; - let remaining_secs = (top_height as f64 - count as f64) * secs_per_hash; - let h = (remaining_secs / 60.0 / 60.0) as u64; - let m = (remaining_secs / 60.0 % 60.0) as u64; - let s = (remaining_secs % 60.0) as u64; - - let block_hash = hex::encode(hash); - let miner_tx_hash = hex::encode(miner_tx_hash); - let prev_hash = hex::encode(prev_hash); - let miner_tx_weight = block.miner_transaction.weight(); - - println!("progress | {count}/{top_height} ({percent:.2}%, {algo}, {bps:.2} blocks/sec, {h}h {m}m {s}s left) +impl Verifier { + fn loop_listen_verify(mut self) { + loop { + let Ok(data) = self.rx.recv() else { + println!("Exiting verify thread {}/{}", self.id, self.thread_count); + return; + }; + + self.verify(data); + } + } + + fn verify(&mut self, data: RpcBlockData) { + //----------------------------------------------- Create panic info. + let p = format!("data: {data:#?}"); + + //----------------------------------------------- Extract data. + let RpcBlockData { + get_block_response, + block, + seed_height, + seed_hash, + txs, + } = data; + let GetBlockResponse { blob, block_header } = get_block_response; + + //----------------------------------------------- Calculate some data. + let calculated_block_reward = block + .miner_transaction + .prefix() + .outputs + .iter() + .map(|o| o.amount.unwrap()) + .sum::(); + let calculated_block_weight = txs + .iter() + .map(|RpcTxData { tx, .. }| tx.weight()) + .sum::(); + let calculated_pow_data = block.serialize_pow_hash(); + let miner_tx_weight = block.miner_transaction.weight(); + + //----------------------------------------------- Verify. + Self::verify_block_properties(&blob, &block, calculated_block_reward, &p); + Self::verify_all_transactions_are_unique(&txs, &p); + Self::verify_transaction_properties(txs, &p); + + self.verify_block_fields( + calculated_block_weight, + calculated_block_reward, + &block, + &p, + block_header, + ); + + let algo = self.verify_pow( + block_header.height, + seed_hash, + block_header.pow_hash, + &calculated_pow_data, + &p, + ); + + //----------------------------------------------- Print progress. + self.print_progress(algo, seed_height, miner_tx_weight, block_header); + } + + fn verify_block_properties( + block_blob: &[u8], + block: &Block, + calculated_block_reward: u64, + p: &str, + ) { + // Test block properties. + assert_eq!(block_blob, block.serialize(), "{p}"); + + assert!( + !block.miner_transaction.prefix().outputs.is_empty(), + "miner_tx has no outputs\n{p}" + ); + + assert_ne!(calculated_block_reward, 0, "block reward is 0\n{p}"); + } + + #[expect(clippy::significant_drop_tightening)] + fn verify_all_transactions_are_unique(txs: &[RpcTxData], p: &str) { + static TX_SET: LazyLock>> = + LazyLock::new(|| Mutex::new(HashSet::new())); + + let mut tx_set = TX_SET.lock().unwrap(); + + for tx_hash in txs.iter().map(|RpcTxData { tx_hash, .. }| tx_hash) { + assert!( + tx_set.insert(*tx_hash), + "duplicated tx_hash: {}, {p}", + hex::encode(tx_hash), + ); + } + } + + fn verify_transaction_properties(txs: Vec, p: &str) { + // Test transaction properties. + for RpcTxData { + tx, + tx_blob, + tx_hash, + } in txs + { + assert_eq!(tx_hash, tx.hash(), "{p}, tx: {tx:#?}"); + assert_ne!(tx.weight(), 0, "{p}, tx: {tx:#?}"); + assert!(!tx.prefix().inputs.is_empty(), "{p}, tx: {tx:#?}"); + assert_eq!(tx_blob, tx.serialize(), "{p}, tx: {tx:#?}"); + assert!(matches!(tx.version(), 1 | 2), "{p}, tx: {tx:#?}"); + } + } + + fn verify_block_fields( + &mut self, + calculated_block_weight: usize, + calculated_block_reward: u64, + block: &Block, + p: &str, + BlockHeader { + block_weight, + hash, + pow_hash: _, + height, + major_version, + minor_version, + miner_tx_hash, + nonce, + num_txes, + prev_hash, + reward, + timestamp, + }: BlockHeader, + ) { + // Test block fields are correct. + assert_eq!(block_weight, calculated_block_weight, "{p}"); + assert_ne!(block.miner_transaction.weight(), 0, "{p}"); + assert_eq!(hash, block.hash(), "{p}"); + assert_eq!( + height, + u64::try_from(block.number().unwrap()).unwrap(), + "{p}" + ); + assert_eq!(major_version, block.header.hardfork_version, "{p}"); + assert_eq!(minor_version, block.header.hardfork_signal, "{p}"); + assert_eq!(miner_tx_hash, block.miner_transaction.hash(), "{p}"); + assert_eq!(nonce, block.header.nonce, "{p}"); + assert_eq!(num_txes, block.transactions.len(), "{p}"); + assert_eq!(prev_hash, block.header.previous, "{p}"); + assert_eq!(reward, calculated_block_reward, "{p}"); + assert_eq!(timestamp, block.header.timestamp, "{p}"); + + if timestamp != 0 { + assert!(timestamp > self.timestamp, "{p}"); + self.timestamp = timestamp; + } + } + + fn verify_pow( + &mut self, + height: u64, + seed_hash: [u8; 32], + pow_hash: [u8; 32], + calculated_pow_data: &[u8], + p: &str, + ) -> &'static str { + let (algo, calculated_pow_hash) = if height < RANDOMX_START_HEIGHT { + CryptoNightHash::hash(calculated_pow_data, height) + } else { + if self.seed_hash != seed_hash { + self.randomx_vm = None; + } + + let randomx_vm = self.randomx_vm.get_or_insert_with(|| { + self.seed_hash = seed_hash; + // crate::randomx::randomx_vm_optimized(&seed_hash) + crate::randomx::randomx_vm_default(&seed_hash) + }); + + let pow_hash = randomx_vm + .calculate_hash(calculated_pow_data) + .unwrap() + .try_into() + .unwrap(); + + ("randomx", pow_hash) + }; + + assert_eq!(calculated_pow_hash, pow_hash, "{p}",); + + algo + } + + #[expect( + clippy::cast_precision_loss, + clippy::cast_possible_truncation, + clippy::cast_sign_loss + )] + fn print_progress( + &self, + algo: &'static str, + seed_height: u64, + miner_tx_weight: usize, + BlockHeader { + block_weight, + hash, + pow_hash, + height, + major_version, + minor_version, + miner_tx_hash, + nonce, + num_txes, + prev_hash, + reward, + timestamp, + }: BlockHeader, + ) { + let count = TESTED_BLOCK_COUNT.fetch_add(1, Ordering::Release) + 1; + let total_tx_count = TESTED_TX_COUNT.fetch_add(num_txes, Ordering::Release) + 1; + + if count % self.update.get() != 0 { + return; + } + + let top_height = self.top_height; + + let percent = (count as f64 / top_height as f64) * 100.0; + + let elapsed = self.now.elapsed().as_secs_f64(); + let secs_per_hash = elapsed / count as f64; + let bps = count as f64 / elapsed; + let remaining_secs = (top_height as f64 - count as f64) * secs_per_hash; + let h = (remaining_secs / 60.0 / 60.0) as u64; + let m = (remaining_secs / 60.0 % 60.0) as u64; + let s = (remaining_secs % 60.0) as u64; + + let pow_hash = hex::encode(pow_hash); + let seed_hash = hex::encode(self.seed_hash); + let block_hash = hex::encode(hash); + let miner_tx_hash = hex::encode(miner_tx_hash); + let prev_hash = hex::encode(prev_hash); + + println!("progress | {count}/{top_height} ({percent:.2}%, {algo}, {bps:.2} blocks/sec, {h}h {m}m {s}s left) seed_hash | {seed_hash} pow_hash | {pow_hash} block_hash | {block_hash} @@ -206,7 +318,5 @@ major_version | {major_version} minor_version | {minor_version} num_txes | {num_txes}\n", ); - } - }); } } From b4797fa31f831aa250e3591577d11181bd8b5eac Mon Sep 17 00:00:00 2001 From: "hinto.janai" Date: Mon, 16 Dec 2024 16:12:18 -0500 Subject: [PATCH 14/16] relative progress, show buffer --- tests/compat/Cargo.toml | 2 +- tests/compat/src/cli.rs | 4 ++- tests/compat/src/main.rs | 45 +++++++++++++++++++++++++++++++++ tests/compat/src/rpc.rs | 9 +++++++ tests/compat/src/types.rs | 3 +++ tests/compat/src/verify.rs | 52 +++++++++++++++++++++++++------------- 6 files changed, 96 insertions(+), 19 deletions(-) diff --git a/tests/compat/Cargo.toml b/tests/compat/Cargo.toml index 9678cfb4..1592cb27 100644 --- a/tests/compat/Cargo.toml +++ b/tests/compat/Cargo.toml @@ -2,7 +2,7 @@ name = "cuprate-tests-compat" version = "0.0.0" edition = "2021" -description = "Compatability tests between `cuprated` and `monerod`" +description = "Compatibility tests between `cuprated` and `monerod`" license = "MIT" authors = ["hinto-janai"] repository = "https://github.com/Cuprate/cuprate/tree/main/tests/compat" diff --git a/tests/compat/src/cli.rs b/tests/compat/src/cli.rs index ed9dda95..b1763230 100644 --- a/tests/compat/src/cli.rs +++ b/tests/compat/src/cli.rs @@ -14,7 +14,9 @@ use clap::Parser; ), )] pub struct Args { - /// Name of the person to greet + /// Base URL to use for `monerod` RPC. + /// + /// This must be a non-restricted RPC. #[arg(short, long, default_value_t = String::from("http://127.0.0.1:18081"))] pub rpc_url: String, diff --git a/tests/compat/src/main.rs b/tests/compat/src/main.rs index c6923e57..a0953f40 100644 --- a/tests/compat/src/main.rs +++ b/tests/compat/src/main.rs @@ -47,3 +47,48 @@ async fn main() { std::thread::sleep(Duration::from_secs(1)); } } + +// some draft code for `monerod` <-> `cuprated` RPC compat testing + +// /// represents a `monerod/cuprated` RPC request type. +// trait RpcRequest { +// /// the expected response type, potentially only being a subset of the fields. +// type SubsetOfResponse: PartialEq; + +// /// create a 'base' request. +// fn base() -> Self; + +// /// permutate the base request into all (or practically) possible requests. +// // e.g. `{"height":0}`, `{"height":1}`, etc +// fn all_possible_inputs_for_rpc_request(self) -> Vec; + +// /// send the request, get the response. +// /// +// /// `monerod` and `cuprated` are both expected to be fully synced. +// fn get(self, node: Node) -> Self::SubsetOfResponse; +// } + +// enum Node { +// Monerod, +// Cuprated, +// } + +// // all RPC requests. +// let all_rpc_requests: Vec = todo!(); + +// // for each request... +// for base in all_rpc_requests { +// // create all possible inputs... +// let requests = all_possible_inputs_for_rpc_request(base); + +// // for each input permutation... +// for r in requests { +// // assert (a potential subset of) `monerod` and `cuprated`'s response fields match in value. +// let monerod_response = r.get(Node::Monerod); +// let cuprated_response = r.get(Node::Cuprated); +// assert_eq!( +// monerod_response.subset_of_response(), +// cuprated_response.subset_of_response(), +// ); +// } +// } diff --git a/tests/compat/src/rpc.rs b/tests/compat/src/rpc.rs index 33153171..9fd1b1dd 100644 --- a/tests/compat/src/rpc.rs +++ b/tests/compat/src/rpc.rs @@ -1,3 +1,5 @@ +use std::time::Instant; + use crossbeam::channel::Sender; use monero_serai::{block::Block, transaction::Transaction}; use rayon::iter::{IndexedParallelIterator, IntoParallelIterator, ParallelIterator}; @@ -144,9 +146,12 @@ impl RpcClient { .collect() } + #[expect(clippy::cast_precision_loss)] pub async fn test(self, top_height: u64, tx: Sender) { use futures::StreamExt; + let now = Instant::now(); + let iter = (0..top_height).map(|height| { let this = self.clone(); let tx = tx.clone(); @@ -185,12 +190,16 @@ impl RpcClient { (seed_height, seed_hash) }; + let elapsed = now.elapsed().as_secs_f64(); + let blocks_per_sec = height as f64 / elapsed; + let data = RpcBlockData { get_block_response, block, seed_height, seed_hash, txs, + blocks_per_sec, }; tx.send(data).unwrap(); diff --git a/tests/compat/src/types.rs b/tests/compat/src/types.rs index 1c9aac27..f164ba78 100644 --- a/tests/compat/src/types.rs +++ b/tests/compat/src/types.rs @@ -20,6 +20,9 @@ pub struct RpcBlockData { /// This vec is: /// - the original transaction blobs pub txs: Vec, + + // Amount of blocks downloaded from RPC per second. + pub blocks_per_sec: f64, } /// Data of a transaction. diff --git a/tests/compat/src/verify.rs b/tests/compat/src/verify.rs index 309fb667..8f1bb37f 100644 --- a/tests/compat/src/verify.rs +++ b/tests/compat/src/verify.rs @@ -23,7 +23,6 @@ struct Verifier { top_height: u64, rx: Receiver, seed_hash: [u8; 32], - timestamp: u64, randomx_vm: Option, } @@ -47,7 +46,6 @@ pub fn spawn_verify_pool( top_height, rx, seed_hash: [0; 32], - timestamp: 0, randomx_vm: None, } .loop_listen_verify(); @@ -78,6 +76,7 @@ impl Verifier { seed_height, seed_hash, txs, + blocks_per_sec, } = data; let GetBlockResponse { blob, block_header } = get_block_response; @@ -101,7 +100,7 @@ impl Verifier { Self::verify_all_transactions_are_unique(&txs, &p); Self::verify_transaction_properties(txs, &p); - self.verify_block_fields( + Self::verify_block_fields( calculated_block_weight, calculated_block_reward, &block, @@ -118,7 +117,13 @@ impl Verifier { ); //----------------------------------------------- Print progress. - self.print_progress(algo, seed_height, miner_tx_weight, block_header); + self.print_progress( + algo, + seed_height, + miner_tx_weight, + blocks_per_sec, + block_header, + ); } fn verify_block_properties( @@ -171,7 +176,6 @@ impl Verifier { } fn verify_block_fields( - &mut self, calculated_block_weight: usize, calculated_block_reward: u64, block: &Block, @@ -208,11 +212,6 @@ impl Verifier { assert_eq!(prev_hash, block.header.previous, "{p}"); assert_eq!(reward, calculated_block_reward, "{p}"); assert_eq!(timestamp, block.header.timestamp, "{p}"); - - if timestamp != 0 { - assert!(timestamp > self.timestamp, "{p}"); - self.timestamp = timestamp; - } } fn verify_pow( @@ -245,7 +244,7 @@ impl Verifier { ("randomx", pow_hash) }; - assert_eq!(calculated_pow_hash, pow_hash, "{p}",); + assert_eq!(calculated_pow_hash, pow_hash, "{p}"); algo } @@ -260,6 +259,7 @@ impl Verifier { algo: &'static str, seed_height: u64, miner_tx_weight: usize, + download_bps: f64, BlockHeader { block_weight, hash, @@ -282,14 +282,29 @@ impl Verifier { return; } - let top_height = self.top_height; + const fn find_count_relative_to_pow_activation(height: u64) -> u64 { + if height <= 1546000 { + height + } else if height <= 1685555 { + height - 1546000 + } else if height <= 1788000 { + height - 1685555 + } else if height <= 1978433 { + height - 1788000 + } else { + height - 1978433 + } + } + let top_height = self.top_height; let percent = (count as f64 / top_height as f64) * 100.0; + let relative_count = find_count_relative_to_pow_activation(height); + let block_buffer = self.rx.len(); let elapsed = self.now.elapsed().as_secs_f64(); - let secs_per_hash = elapsed / count as f64; - let bps = count as f64 / elapsed; - let remaining_secs = (top_height as f64 - count as f64) * secs_per_hash; + let secs_per_hash = elapsed / relative_count as f64; + let verify_bps = relative_count as f64 / elapsed; + let remaining_secs = (top_height as f64 - relative_count as f64) * secs_per_hash; let h = (remaining_secs / 60.0 / 60.0) as u64; let m = (remaining_secs / 60.0 % 60.0) as u64; let s = (remaining_secs % 60.0) as u64; @@ -300,7 +315,10 @@ impl Verifier { let miner_tx_hash = hex::encode(miner_tx_hash); let prev_hash = hex::encode(prev_hash); - println!("progress | {count}/{top_height} ({percent:.2}%, {algo}, {bps:.2} blocks/sec, {h}h {m}m {s}s left) + println!( + "progress | {count}/{top_height} ({percent:.2}%) +remaining | [{h}h {m}m {s}s] left for [{algo}] +block_pipeline | [download {download_bps:.2}/sec] -> [buffer {block_buffer}] -> [verify {verify_bps:.2}/sec] seed_hash | {seed_hash} pow_hash | {pow_hash} block_hash | {block_hash} @@ -317,6 +335,6 @@ miner_tx_weight | {miner_tx_weight} major_version | {major_version} minor_version | {minor_version} num_txes | {num_txes}\n", - ); + ); } } From 1bc5e4721c56db2eec2d88760fdee3f581962c9a Mon Sep 17 00:00:00 2001 From: "hinto.janai" Date: Mon, 16 Dec 2024 22:14:51 -0500 Subject: [PATCH 15/16] more cli args --- tests/compat/src/cli.rs | 16 +++++++++++++--- tests/compat/src/main.rs | 10 ++++++++-- tests/compat/src/rpc.rs | 8 +++++--- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/tests/compat/src/cli.rs b/tests/compat/src/cli.rs index b1763230..1ddaaaec 100644 --- a/tests/compat/src/cli.rs +++ b/tests/compat/src/cli.rs @@ -17,15 +17,25 @@ pub struct Args { /// Base URL to use for `monerod` RPC. /// /// This must be a non-restricted RPC. - #[arg(short, long, default_value_t = String::from("http://127.0.0.1:18081"))] + #[arg(long, default_value_t = String::from("http://127.0.0.1:18081"))] pub rpc_url: String, + /// Amount of async RPC tasks to spawn. + #[arg(long, default_value_t = NonZeroUsize::new(4).unwrap())] + pub rpc_tasks: NonZeroUsize, + + /// The maximum capacity of the block buffer in-between the RPC and verifier. + /// + /// `0` will cause the buffer to be unbounded. + #[arg(long, default_value_t = 1000)] + pub buffer_limit: usize, + /// Amount of verifying threads to spawn. - #[arg(short, long, default_value_t = std::thread::available_parallelism().unwrap())] + #[arg(long, default_value_t = std::thread::available_parallelism().unwrap())] pub threads: NonZeroUsize, /// Print an update every `update` amount of blocks. - #[arg(short, long, default_value_t = NonZeroU64::new(500).unwrap())] + #[arg(long, default_value_t = NonZeroU64::new(500).unwrap())] pub update: NonZeroU64, } diff --git a/tests/compat/src/main.rs b/tests/compat/src/main.rs index a0953f40..69384aa9 100644 --- a/tests/compat/src/main.rs +++ b/tests/compat/src/main.rs @@ -21,17 +21,23 @@ async fn main() { let cli::Args { rpc_url, update, + rpc_tasks, + buffer_limit, threads, } = cli::Args::get(); // Set-up RPC client. - let client = rpc::RpcClient::new(rpc_url).await; + let client = rpc::RpcClient::new(rpc_url, rpc_tasks).await; let top_height = client.top_height; println!("top_height: {top_height}"); println!(); // Test. - let (tx, rx) = crossbeam::channel::unbounded(); + let (tx, rx) = if buffer_limit == 0 { + crossbeam::channel::unbounded() + } else { + crossbeam::channel::bounded(buffer_limit) + }; verify::spawn_verify_pool(threads, update, top_height, rx); client.test(top_height, tx).await; diff --git a/tests/compat/src/rpc.rs b/tests/compat/src/rpc.rs index 9fd1b1dd..42194557 100644 --- a/tests/compat/src/rpc.rs +++ b/tests/compat/src/rpc.rs @@ -1,4 +1,4 @@ -use std::time::Instant; +use std::{num::NonZeroUsize, time::Instant}; use crossbeam::channel::Sender; use monero_serai::{block::Block, transaction::Transaction}; @@ -20,11 +20,12 @@ pub struct RpcClient { client: Client, json_rpc_url: String, get_transactions_url: String, + rpc_tasks: NonZeroUsize, pub top_height: u64, } impl RpcClient { - pub async fn new(rpc_url: String) -> Self { + pub async fn new(rpc_url: String, rpc_tasks: NonZeroUsize) -> Self { let headers = { let mut h = HeaderMap::new(); h.insert("Content-Type", HeaderValue::from_static("application/json")); @@ -70,6 +71,7 @@ impl RpcClient { client, json_rpc_url, get_transactions_url, + rpc_tasks, top_height, } } @@ -207,7 +209,7 @@ impl RpcClient { }); futures::stream::iter(iter) - .buffer_unordered(4) // This can't be too high or else we get bottlenecked by `monerod` + .buffer_unordered(self.rpc_tasks.get()) // This can't be too high or else we get bottlenecked by `monerod` .for_each(|()| async {}) .await; } From ce6807546c8c6876f2e8685f4ebe9dd0379660bc Mon Sep 17 00:00:00 2001 From: "hinto.janai" Date: Tue, 17 Dec 2024 17:45:53 -0500 Subject: [PATCH 16/16] fixes --- tests/compat/src/verify.rs | 160 ++++++++++++++++--------------------- 1 file changed, 67 insertions(+), 93 deletions(-) diff --git a/tests/compat/src/verify.rs b/tests/compat/src/verify.rs index 8f1bb37f..6c8a21de 100644 --- a/tests/compat/src/verify.rs +++ b/tests/compat/src/verify.rs @@ -17,7 +17,7 @@ use crate::{ struct Verifier { id: usize, - now: Instant, + start_of_new_pow: Instant, thread_count: NonZeroUsize, update: NonZeroU64, top_height: u64, @@ -33,14 +33,12 @@ pub fn spawn_verify_pool( top_height: u64, rx: Receiver, ) { - let now = Instant::now(); - - for id in 0..thread_count.get() { + for id in 1..=thread_count.get() { let rx = rx.clone(); std::thread::spawn(move || { Verifier { id, - now, + start_of_new_pow: Instant::now(), thread_count, update, top_height, @@ -80,39 +78,14 @@ impl Verifier { } = data; let GetBlockResponse { blob, block_header } = get_block_response; - //----------------------------------------------- Calculate some data. - let calculated_block_reward = block - .miner_transaction - .prefix() - .outputs - .iter() - .map(|o| o.amount.unwrap()) - .sum::(); - let calculated_block_weight = txs - .iter() - .map(|RpcTxData { tx, .. }| tx.weight()) - .sum::(); - let calculated_pow_data = block.serialize_pow_hash(); - let miner_tx_weight = block.miner_transaction.weight(); - //----------------------------------------------- Verify. - Self::verify_block_properties(&blob, &block, calculated_block_reward, &p); - Self::verify_all_transactions_are_unique(&txs, &p); - Self::verify_transaction_properties(txs, &p); - - Self::verify_block_fields( - calculated_block_weight, - calculated_block_reward, - &block, - &p, - block_header, - ); - + Self::verify_blocks(&blob, &block, &txs, &p, block_header); + Self::verify_transactions(txs, &p); let algo = self.verify_pow( block_header.height, seed_hash, block_header.pow_hash, - &calculated_pow_data, + &block.serialize_pow_hash(), &p, ); @@ -120,65 +93,16 @@ impl Verifier { self.print_progress( algo, seed_height, - miner_tx_weight, + block.miner_transaction.weight(), blocks_per_sec, block_header, ); } - fn verify_block_properties( + fn verify_blocks( block_blob: &[u8], block: &Block, - calculated_block_reward: u64, - p: &str, - ) { - // Test block properties. - assert_eq!(block_blob, block.serialize(), "{p}"); - - assert!( - !block.miner_transaction.prefix().outputs.is_empty(), - "miner_tx has no outputs\n{p}" - ); - - assert_ne!(calculated_block_reward, 0, "block reward is 0\n{p}"); - } - - #[expect(clippy::significant_drop_tightening)] - fn verify_all_transactions_are_unique(txs: &[RpcTxData], p: &str) { - static TX_SET: LazyLock>> = - LazyLock::new(|| Mutex::new(HashSet::new())); - - let mut tx_set = TX_SET.lock().unwrap(); - - for tx_hash in txs.iter().map(|RpcTxData { tx_hash, .. }| tx_hash) { - assert!( - tx_set.insert(*tx_hash), - "duplicated tx_hash: {}, {p}", - hex::encode(tx_hash), - ); - } - } - - fn verify_transaction_properties(txs: Vec, p: &str) { - // Test transaction properties. - for RpcTxData { - tx, - tx_blob, - tx_hash, - } in txs - { - assert_eq!(tx_hash, tx.hash(), "{p}, tx: {tx:#?}"); - assert_ne!(tx.weight(), 0, "{p}, tx: {tx:#?}"); - assert!(!tx.prefix().inputs.is_empty(), "{p}, tx: {tx:#?}"); - assert_eq!(tx_blob, tx.serialize(), "{p}, tx: {tx:#?}"); - assert!(matches!(tx.version(), 1 | 2), "{p}, tx: {tx:#?}"); - } - } - - fn verify_block_fields( - calculated_block_weight: usize, - calculated_block_reward: u64, - block: &Block, + txs: &[RpcTxData], p: &str, BlockHeader { block_weight, @@ -195,15 +119,28 @@ impl Verifier { timestamp, }: BlockHeader, ) { - // Test block fields are correct. - assert_eq!(block_weight, calculated_block_weight, "{p}"); + let calculated_block_reward = block + .miner_transaction + .prefix() + .outputs + .iter() + .map(|o| o.amount.unwrap()) + .sum::(); + + let calculated_block_weight = txs + .iter() + .map(|RpcTxData { tx, .. }| tx.weight()) + .sum::(); + + let block_number = u64::try_from(block.number().unwrap()).unwrap(); + + assert!(!block.miner_transaction.prefix().outputs.is_empty(), "{p}"); + assert_ne!(calculated_block_reward, 0, "{p}"); assert_ne!(block.miner_transaction.weight(), 0, "{p}"); + assert_eq!(block_blob, block.serialize(), "{p}"); + assert_eq!(block_weight, calculated_block_weight, "{p}"); assert_eq!(hash, block.hash(), "{p}"); - assert_eq!( - height, - u64::try_from(block.number().unwrap()).unwrap(), - "{p}" - ); + assert_eq!(height, block_number, "{p}"); assert_eq!(major_version, block.header.hardfork_version, "{p}"); assert_eq!(minor_version, block.header.hardfork_signal, "{p}"); assert_eq!(miner_tx_hash, block.miner_transaction.hash(), "{p}"); @@ -214,6 +151,39 @@ impl Verifier { assert_eq!(timestamp, block.header.timestamp, "{p}"); } + fn verify_transactions(txs: Vec, p: &str) { + Self::verify_all_transactions_are_unique(&txs, p); + + for RpcTxData { + tx, + tx_blob, + tx_hash, + } in txs + { + assert_eq!(tx_hash, tx.hash(), "{p}, tx: {tx:#?}"); + assert_ne!(tx.weight(), 0, "{p}, tx: {tx:#?}"); + assert!(!tx.prefix().inputs.is_empty(), "{p}, tx: {tx:#?}"); + assert_eq!(tx_blob, tx.serialize(), "{p}, tx: {tx:#?}"); + assert!(matches!(tx.version(), 1 | 2), "{p}, tx: {tx:#?}"); + } + } + + #[expect(clippy::significant_drop_tightening)] + fn verify_all_transactions_are_unique(txs: &[RpcTxData], p: &str) { + static TX_SET: LazyLock>> = + LazyLock::new(|| Mutex::new(HashSet::new())); + + let mut tx_set = TX_SET.lock().unwrap(); + + for tx_hash in txs.iter().map(|RpcTxData { tx_hash, .. }| tx_hash) { + assert!( + tx_set.insert(*tx_hash), + "duplicated tx_hash: {}, {p}", + hex::encode(tx_hash), + ); + } + } + fn verify_pow( &mut self, height: u64, @@ -222,6 +192,10 @@ impl Verifier { calculated_pow_data: &[u8], p: &str, ) -> &'static str { + if matches!(height, 1546000 | 1685555 | 1788000 | 1978433) { + self.start_of_new_pow = Instant::now(); + } + let (algo, calculated_pow_hash) = if height < RANDOMX_START_HEIGHT { CryptoNightHash::hash(calculated_pow_data, height) } else { @@ -301,7 +275,7 @@ impl Verifier { let relative_count = find_count_relative_to_pow_activation(height); let block_buffer = self.rx.len(); - let elapsed = self.now.elapsed().as_secs_f64(); + let elapsed = self.start_of_new_pow.elapsed().as_secs_f64(); let secs_per_hash = elapsed / relative_count as f64; let verify_bps = relative_count as f64 / elapsed; let remaining_secs = (top_height as f64 - relative_count as f64) * secs_per_hash;