From 743fcce2b8a2094efcd9bf7343857d22dffbe569 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Tue, 17 Oct 2023 13:03:47 -0300 Subject: [PATCH 01/36] get started with the blockchain scanner poc --- Cargo.lock | 155 +++++++++++++++++++++++++++++--- Cargo.toml | 1 + zebra-scanner/Cargo.toml | 35 ++++++++ zebra-scanner/src/lib.rs | 8 ++ zebra-scanner/src/tests.rs | 176 +++++++++++++++++++++++++++++++++++++ 5 files changed, 362 insertions(+), 13 deletions(-) create mode 100644 zebra-scanner/Cargo.toml create mode 100644 zebra-scanner/src/lib.rs create mode 100644 zebra-scanner/src/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 9b8d40c3b34..b18ca29b646 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -546,7 +546,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a813dadc684e4c78a4547757debd99666282545d90e4ccc3210913ed4337ad2" dependencies = [ - "incrementalmerkletree", + "incrementalmerkletree 0.4.0", ] [[package]] @@ -2020,6 +2020,15 @@ dependencies = [ "either", ] +[[package]] +name = "incrementalmerkletree" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "361c467824d4d9d4f284be4b2608800839419dccc4d4608f28345237fe354623" +dependencies = [ + "either", +] + [[package]] name = "indenter" version = "0.3.3" @@ -2714,7 +2723,35 @@ dependencies = [ "halo2_gadgets", "halo2_proofs", "hex", - "incrementalmerkletree", + "incrementalmerkletree 0.4.0", + "lazy_static", + "memuse", + "nonempty", + "pasta_curves", + "rand 0.8.5", + "reddsa", + "serde", + "subtle", + "tracing", + "zcash_note_encryption", +] + +[[package]] +name = "orchard" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d31e68534df32024dcc89a8390ec6d7bef65edd87d91b45cfb481a2eb2d77c5" +dependencies = [ + "aes", + "bitvec", + "blake2b_simd", + "ff", + "fpe", + "group", + "halo2_gadgets", + "halo2_proofs", + "hex", + "incrementalmerkletree 0.5.0", "lazy_static", "memuse", "nonempty", @@ -4029,6 +4066,18 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shardtree" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c19f96dde3a8693874f7e7c53d95616569b4009379a903789efbd448f4ea9cc7" +dependencies = [ + "bitflags 2.4.0", + "either", + "incrementalmerkletree 0.5.0", + "tracing", +] + [[package]] name = "shlex" version = "1.2.0" @@ -5392,6 +5441,39 @@ dependencies = [ "zcash_encoding", ] +[[package]] +name = "zcash_client_backend" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6a382af39be9ee5a3788157145c404b7cd19acc440903f6c34b09fb44f0e991" +dependencies = [ + "base64 0.21.4", + "bech32", + "bls12_381", + "bs58", + "crossbeam-channel", + "group", + "hex", + "incrementalmerkletree 0.5.0", + "memuse", + "nom", + "orchard 0.6.0", + "percent-encoding", + "prost", + "rayon", + "secrecy", + "shardtree", + "subtle", + "time", + "tonic-build", + "tracing", + "which", + "zcash_address", + "zcash_encoding", + "zcash_note_encryption", + "zcash_primitives 0.13.0", +] + [[package]] name = "zcash_encoding" version = "0.2.0" @@ -5445,12 +5527,12 @@ dependencies = [ "group", "hdwallet", "hex", - "incrementalmerkletree", + "incrementalmerkletree 0.4.0", "jubjub", "lazy_static", "memuse", "nonempty", - "orchard", + "orchard 0.5.0", "rand 0.8.5", "rand_core 0.6.4", "ripemd", @@ -5462,6 +5544,39 @@ dependencies = [ "zcash_note_encryption", ] +[[package]] +name = "zcash_primitives" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d17e4c94ca8d69d2fcf2be97522da5732a580eb2125cda3b150761952f8df8e6" +dependencies = [ + "aes", + "bip0039", + "bitvec", + "blake2b_simd", + "blake2s_simd", + "bls12_381", + "byteorder", + "equihash", + "ff", + "fpe", + "group", + "hex", + "incrementalmerkletree 0.5.0", + "jubjub", + "lazy_static", + "memuse", + "nonempty", + "orchard 0.6.0", + "rand 0.8.5", + "rand_core 0.6.4", + "sha2", + "subtle", + "zcash_address", + "zcash_encoding", + "zcash_note_encryption", +] + [[package]] name = "zcash_proofs" version = "0.12.1" @@ -5473,7 +5588,7 @@ dependencies = [ "bls12_381", "group", "home", - "incrementalmerkletree", + "incrementalmerkletree 0.4.0", "jubjub", "known-folders", "lazy_static", @@ -5482,7 +5597,7 @@ dependencies = [ "redjubjub", "tracing", "xdg", - "zcash_primitives", + "zcash_primitives 0.12.0", ] [[package]] @@ -5502,12 +5617,12 @@ dependencies = [ "cxx", "cxx-gen", "group", - "incrementalmerkletree", + "incrementalmerkletree 0.4.0", "jubjub", "libc", "memuse", "metrics", - "orchard", + "orchard 0.5.0", "rand 0.8.5", "rand_core 0.6.4", "rayon", @@ -5517,7 +5632,7 @@ dependencies = [ "zcash_address", "zcash_encoding", "zcash_note_encryption", - "zcash_primitives", + "zcash_primitives 0.12.0", "zcash_proofs", ] @@ -5544,12 +5659,12 @@ dependencies = [ "halo2_proofs", "hex", "humantime", - "incrementalmerkletree", + "incrementalmerkletree 0.4.0", "itertools 0.11.0", "jubjub", "lazy_static", "num-integer", - "orchard", + "orchard 0.5.0", "primitive-types", "proptest", "proptest-derive", @@ -5578,7 +5693,7 @@ dependencies = [ "zcash_encoding", "zcash_history", "zcash_note_encryption", - "zcash_primitives", + "zcash_primitives 0.12.0", "zebra-test", ] @@ -5602,7 +5717,7 @@ dependencies = [ "metrics", "num-integer", "once_cell", - "orchard", + "orchard 0.5.0", "proptest", "proptest-derive", "rand 0.8.5", @@ -5712,6 +5827,20 @@ dependencies = [ "zebra-test", ] +[[package]] +name = "zebra-scanner" +version = "1.0.0-beta.29" +dependencies = [ + "bls12_381", + "ff", + "group", + "jubjub", + "rand 0.8.5", + "zcash_client_backend", + "zcash_note_encryption", + "zcash_primitives 0.13.0", +] + [[package]] name = "zebra-script" version = "1.0.0-beta.29" diff --git a/Cargo.toml b/Cargo.toml index 0f81f34fa45..ea2e23db201 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ members = [ "zebra-node-services", "zebra-test", "zebra-utils", + "zebra-scanner", "tower-batch-control", "tower-fallback", ] diff --git a/zebra-scanner/Cargo.toml b/zebra-scanner/Cargo.toml new file mode 100644 index 00000000000..a147dfc22d6 --- /dev/null +++ b/zebra-scanner/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "zebra-scanner" +version = "1.0.0-beta.29" +authors = ["Zcash Foundation "] +description = "Blockchain scanner for the Zebra Zcash node" +license = "MIT OR Apache-2.0" +repository = "https://github.com/ZcashFoundation/zebra" +edition = "2021" + +readme = "../README.md" +homepage = "https://zfnd.org/zebra/" +# crates.io is limited to 5 keywords and categories +keywords = ["zebra", "zcash"] +# Must be one of +categories = ["cryptography::cryptocurrencies"] + +[features] + +# Production features that activate extra dependencies, or extra features in dependencies + +[dependencies] +zcash_client_backend = "0.10.0" +zcash_primitives = "0.13.0" +zcash_note_encryption = "0.4.0" + +rand = "0.8.5" + +bls12_381 = "0.8.0" +jubjub = "0.10.0" +ff = "0.13.0" +group = "0.13.0" + +# zebra-chain = { path = "../zebra-chain", version = "1.0.0-beta.29", features = ["async-error"] } + +[dev-dependencies] diff --git a/zebra-scanner/src/lib.rs b/zebra-scanner/src/lib.rs new file mode 100644 index 00000000000..ec988b34dae --- /dev/null +++ b/zebra-scanner/src/lib.rs @@ -0,0 +1,8 @@ +//! Blockchain scanner code for Zebra. 🦓 + +#![doc(html_favicon_url = "https://zfnd.org/wp-content/uploads/2022/03/zebra-favicon-128.png")] +#![doc(html_logo_url = "https://zfnd.org/wp-content/uploads/2022/03/zebra-icon.png")] +#![doc(html_root_url = "https://doc.zebra.zfnd.org/zebra_state")] + +#[cfg(test)] +mod tests; diff --git a/zebra-scanner/src/tests.rs b/zebra-scanner/src/tests.rs new file mode 100644 index 00000000000..595d2d6995e --- /dev/null +++ b/zebra-scanner/src/tests.rs @@ -0,0 +1,176 @@ +//! Test that we can scan blocks. +//! + +use zcash_client_backend::proto::compact_formats::{ + self as compact, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx, +}; +use zcash_note_encryption::Domain; +use zcash_primitives::{ + block::BlockHash, + consensus::{BlockHeight, Network}, + constants::SPENDING_KEY_GENERATOR, + memo::MemoBytes, + sapling::{ + note_encryption::{sapling_note_encryption, SaplingDomain}, + util::generate_random_rseed, + value::NoteValue, + Note, Nullifier, SaplingIvk, + }, + zip32::{AccountId, DiversifiableFullViewingKey, ExtendedSpendingKey}, +}; + +use rand::{rngs::OsRng, RngCore}; + +use ff::{Field, PrimeField}; +use group::GroupEncoding; + +#[test] +fn scanning_from_zebra() { + let account = AccountId::from(12); + let extsk = ExtendedSpendingKey::master(&[]); + let dfvk = extsk.to_diversifiable_full_viewing_key(); + + let nf = Nullifier([7; 32]); + + let cb = fake_compact_block( + 1u32.into(), + BlockHash([0; 32]), + nf, + &dfvk, + 1, + false, + Some(0), + ); + + assert_eq!(cb.vtx.len(), 2); + + let vks: Vec<(&AccountId, &SaplingIvk)> = vec![]; + let network = zcash_primitives::consensus::TestNetwork; + + let res = + zcash_client_backend::scanning::scan_block(&network, cb, &vks[..], &[(account, nf)], None) + .unwrap(); + + dbg!(res.transactions().len()); + assert_eq!(res.metadata().block_height(), BlockHeight::from(1)); + +} + +// This is a copy of zcash_primitives `fake_compact_block` where the `value` argument was changed to +// be a number for easier conversion: +// https://github.com/zcash/librustzcash/blob/zcash_primitives-0.13.0/zcash_client_backend/src/scanning.rs#L635 +// We need to copy because this is a test private function upstream. +fn fake_compact_block( + height: BlockHeight, + prev_hash: BlockHash, + nf: Nullifier, + dfvk: &DiversifiableFullViewingKey, + value: u64, + tx_after: bool, + initial_sapling_tree_size: Option, +) -> CompactBlock { + let to = dfvk.default_address().1; + + // Create a fake Note for the account + let mut rng = OsRng; + let rseed = generate_random_rseed(&Network::TestNetwork, height, &mut rng); + let note = Note::from_parts(to, NoteValue::from_raw(value), rseed); + let encryptor = sapling_note_encryption::<_, Network>( + Some(dfvk.fvk().ovk), + note.clone(), + MemoBytes::empty(), + &mut rng, + ); + let cmu = note.cmu().to_bytes().to_vec(); + let ephemeral_key = SaplingDomain::::epk_bytes(encryptor.epk()) + .0 + .to_vec(); + let enc_ciphertext = encryptor.encrypt_note_plaintext(); + + // Create a fake CompactBlock containing the note + let mut cb = CompactBlock { + hash: { + let mut hash = vec![0; 32]; + rng.fill_bytes(&mut hash); + hash + }, + prev_hash: prev_hash.0.to_vec(), + height: height.into(), + ..Default::default() + }; + + // Add a random Sapling tx before ours + { + let mut tx = random_compact_tx(&mut rng); + tx.index = cb.vtx.len() as u64; + cb.vtx.push(tx); + } + + let cspend = CompactSaplingSpend { nf: nf.0.to_vec() }; + let cout = CompactSaplingOutput { + cmu, + ephemeral_key, + ciphertext: enc_ciphertext.as_ref()[..52].to_vec(), + }; + let mut ctx = CompactTx::default(); + let mut txid = vec![0; 32]; + rng.fill_bytes(&mut txid); + ctx.hash = txid; + ctx.spends.push(cspend); + ctx.outputs.push(cout); + ctx.index = cb.vtx.len() as u64; + cb.vtx.push(ctx); + + // Optionally add another random Sapling tx after ours + if tx_after { + let mut tx = random_compact_tx(&mut rng); + tx.index = cb.vtx.len() as u64; + cb.vtx.push(tx); + } + + cb.chain_metadata = initial_sapling_tree_size.map(|s| compact::ChainMetadata { + sapling_commitment_tree_size: s + cb + .vtx + .iter() + .map(|tx| tx.outputs.len() as u32) + .sum::(), + ..Default::default() + }); + + cb +} + +// This is an exact copy of `zcash_client_backend::scanning::random_compact_tx`: +// https://github.com/zcash/librustzcash/blob/zcash_primitives-0.13.0/zcash_client_backend/src/scanning.rs#L597 +// We need to copy because this is a test private function upstream. +fn random_compact_tx(mut rng: impl RngCore) -> CompactTx { + let fake_nf = { + let mut nf = vec![0; 32]; + rng.fill_bytes(&mut nf); + nf + }; + let fake_cmu = { + let fake_cmu = bls12_381::Scalar::random(&mut rng); + fake_cmu.to_repr().as_ref().to_owned() + }; + let fake_epk = { + let mut buffer = [0; 64]; + rng.fill_bytes(&mut buffer); + let fake_esk = jubjub::Fr::from_bytes_wide(&buffer); + let fake_epk = SPENDING_KEY_GENERATOR * fake_esk; + fake_epk.to_bytes().to_vec() + }; + let cspend = CompactSaplingSpend { nf: fake_nf }; + let cout = CompactSaplingOutput { + cmu: fake_cmu, + ephemeral_key: fake_epk, + ciphertext: vec![0; 52], + }; + let mut ctx = CompactTx::default(); + let mut txid = vec![0; 32]; + rng.fill_bytes(&mut txid); + ctx.hash = txid; + ctx.spends.push(cspend); + ctx.outputs.push(cout); + ctx +} From d6600a919316ca3bbf88116384acc2cc920fea4c Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Tue, 17 Oct 2023 16:31:47 -0300 Subject: [PATCH 02/36] rustfmt --- Cargo.lock | 2 +- zebra-scanner/src/tests.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f2bf28992be..242d65ad9fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4051,7 +4051,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c19f96dde3a8693874f7e7c53d95616569b4009379a903789efbd448f4ea9cc7" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.1", "either", "incrementalmerkletree 0.5.0", "tracing", diff --git a/zebra-scanner/src/tests.rs b/zebra-scanner/src/tests.rs index 595d2d6995e..1a8b495fd39 100644 --- a/zebra-scanner/src/tests.rs +++ b/zebra-scanner/src/tests.rs @@ -53,7 +53,6 @@ fn scanning_from_zebra() { dbg!(res.transactions().len()); assert_eq!(res.metadata().block_height(), BlockHeight::from(1)); - } // This is a copy of zcash_primitives `fake_compact_block` where the `value` argument was changed to From cb1a76a194785c4ad4b6df528671ccea33da8985 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Tue, 17 Oct 2023 16:46:48 -0300 Subject: [PATCH 03/36] fix the tests --- zebra-scanner/src/tests.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/zebra-scanner/src/tests.rs b/zebra-scanner/src/tests.rs index 1a8b495fd39..439578ea241 100644 --- a/zebra-scanner/src/tests.rs +++ b/zebra-scanner/src/tests.rs @@ -42,6 +42,7 @@ fn scanning_from_zebra() { Some(0), ); + // The fake block function will have our transaction and a random one. assert_eq!(cb.vtx.len(), 2); let vks: Vec<(&AccountId, &SaplingIvk)> = vec![]; @@ -51,8 +52,8 @@ fn scanning_from_zebra() { zcash_client_backend::scanning::scan_block(&network, cb, &vks[..], &[(account, nf)], None) .unwrap(); - dbg!(res.transactions().len()); - assert_eq!(res.metadata().block_height(), BlockHeight::from(1)); + // The response should have one transaction relevant to the key we provided. + assert_eq!(res.transactions().len(), 1); } // This is a copy of zcash_primitives `fake_compact_block` where the `value` argument was changed to From c0f7dbd634eb20f94cc9d16750c894f5104b5e21 Mon Sep 17 00:00:00 2001 From: arya2 Date: Tue, 17 Oct 2023 19:28:41 -0400 Subject: [PATCH 04/36] Reads blocks from db --- Cargo.lock | 5 +++++ zebra-scanner/Cargo.toml | 9 ++++++++- zebra-scanner/src/tests.rs | 39 +++++++++++++++++++++++++++++++------- 3 files changed, 45 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 242d65ad9fa..71fc57f0d8a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5801,13 +5801,18 @@ name = "zebra-scanner" version = "1.0.0-beta.29" dependencies = [ "bls12_381", + "color-eyre", "ff", "group", "jubjub", "rand 0.8.5", + "tokio", "zcash_client_backend", "zcash_note_encryption", "zcash_primitives 0.13.0", + "zebra-chain", + "zebra-consensus", + "zebra-state", ] [[package]] diff --git a/zebra-scanner/Cargo.toml b/zebra-scanner/Cargo.toml index a147dfc22d6..e7c026c07c4 100644 --- a/zebra-scanner/Cargo.toml +++ b/zebra-scanner/Cargo.toml @@ -30,6 +30,13 @@ jubjub = "0.10.0" ff = "0.13.0" group = "0.13.0" -# zebra-chain = { path = "../zebra-chain", version = "1.0.0-beta.29", features = ["async-error"] } +tokio = { version = "1.33.0", features = ["full", "tracing", "test-util"] } + +zebra-state = { path = "../zebra-state", version = "1.0.0-beta.30" } +zebra-consensus = { path = "../zebra-consensus", version = "1.0.0-beta.30" } + +zebra-chain = { path = "../zebra-chain", version = "1.0.0-beta.30", features = ["async-error"] } [dev-dependencies] + +color-eyre = { version = "0.6.2" } diff --git a/zebra-scanner/src/tests.rs b/zebra-scanner/src/tests.rs index 439578ea241..5bc4153a265 100644 --- a/zebra-scanner/src/tests.rs +++ b/zebra-scanner/src/tests.rs @@ -1,8 +1,11 @@ //! Test that we can scan blocks. //! -use zcash_client_backend::proto::compact_formats::{ - self as compact, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx, +use zcash_client_backend::{ + proto::compact_formats::{ + self as compact, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx, + }, + scanning::scan_block, }; use zcash_note_encryption::Domain; use zcash_primitives::{ @@ -19,13 +22,35 @@ use zcash_primitives::{ zip32::{AccountId, DiversifiableFullViewingKey, ExtendedSpendingKey}, }; +use color_eyre::Result; + use rand::{rngs::OsRng, RngCore}; use ff::{Field, PrimeField}; use group::GroupEncoding; +use zebra_chain::block::Height; + +#[tokio::test] +async fn scanning_from_zebra() -> Result<()> { + let (consensus_config, state_config, network) = Default::default(); + + let (_, max_checkpoint_height) = + zebra_consensus::router::init_checkpoint_list(consensus_config, network); + + let (_state_service, read_only_state_service, _latest_chain_tip, _chain_tip_change) = + zebra_state::spawn_init(state_config, network, max_checkpoint_height, 3000).await?; + + let db = read_only_state_service.db(); + let mut height = Height(0); + + while let Some(_block) = db.block(height.into()) { + // TODO: Convert block to compact block, scan, and update state + + // let res = scan_block(&network, cb, &vks[..], &[(account, nf)], None).unwrap(); + + height = height.next()?; + } -#[test] -fn scanning_from_zebra() { let account = AccountId::from(12); let extsk = ExtendedSpendingKey::master(&[]); let dfvk = extsk.to_diversifiable_full_viewing_key(); @@ -48,12 +73,12 @@ fn scanning_from_zebra() { let vks: Vec<(&AccountId, &SaplingIvk)> = vec![]; let network = zcash_primitives::consensus::TestNetwork; - let res = - zcash_client_backend::scanning::scan_block(&network, cb, &vks[..], &[(account, nf)], None) - .unwrap(); + let res = scan_block(&network, cb, &vks[..], &[(account, nf)], None).unwrap(); // The response should have one transaction relevant to the key we provided. assert_eq!(res.transactions().len(), 1); + + Ok(()) } // This is a copy of zcash_primitives `fake_compact_block` where the `value` argument was changed to From c5c6c805962cbe1325176fed3e90bf468360ace2 Mon Sep 17 00:00:00 2001 From: arya2 Date: Tue, 17 Oct 2023 20:21:30 -0400 Subject: [PATCH 05/36] Adds conversion functions --- zebra-scanner/src/tests.rs | 111 +++++++++++++++++++++++++++++++++++-- 1 file changed, 107 insertions(+), 4 deletions(-) diff --git a/zebra-scanner/src/tests.rs b/zebra-scanner/src/tests.rs index 5bc4153a265..8e6f2c22c48 100644 --- a/zebra-scanner/src/tests.rs +++ b/zebra-scanner/src/tests.rs @@ -1,9 +1,12 @@ //! Test that we can scan blocks. //! +use std::sync::Arc; + use zcash_client_backend::{ proto::compact_formats::{ - self as compact, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx, + self as compact, ChainMetadata, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, + CompactTx, }, scanning::scan_block, }; @@ -28,7 +31,11 @@ use rand::{rngs::OsRng, RngCore}; use ff::{Field, PrimeField}; use group::GroupEncoding; -use zebra_chain::block::Height; +use zebra_chain::{ + block::{Block, Height}, + serialization::ZcashSerialize, + transaction::Transaction, +}; #[tokio::test] async fn scanning_from_zebra() -> Result<()> { @@ -43,8 +50,29 @@ async fn scanning_from_zebra() -> Result<()> { let db = read_only_state_service.db(); let mut height = Height(0); - while let Some(_block) = db.block(height.into()) { - // TODO: Convert block to compact block, scan, and update state + while let Some(block) = db.block(height.into()) { + // Get chain metadata + let sapling_tree_size = db + .sapling_tree_by_hash_or_height(height.into()) + .expect("should exist") + .count(); + let orchard_tree_size = db + .orchard_tree_by_hash_or_height(height.into()) + .expect("should exist") + .count(); + + let chain_metadata = ChainMetadata { + sapling_commitment_tree_size: sapling_tree_size + .try_into() + .expect("position should fit in u32"), + orchard_commitment_tree_size: orchard_tree_size + .try_into() + .expect("position should fit in u32"), + }; + + let _compact_block = block_to_compact(block, chain_metadata); + + // TODO: scan block and update state // let res = scan_block(&network, cb, &vks[..], &[(account, nf)], None).unwrap(); @@ -81,6 +109,81 @@ async fn scanning_from_zebra() -> Result<()> { Ok(()) } +fn block_to_compact(block: Arc, chain_metadata: ChainMetadata) -> CompactBlock { + CompactBlock { + height: block + .coinbase_height() + .expect("verified block should have a valid height") + .0 + .into(), + hash: block.hash().bytes_in_display_order().to_vec(), + prev_hash: block + .header + .previous_block_hash + .bytes_in_display_order() + .to_vec(), + time: block + .header + .time + .timestamp() + .try_into() + .expect("should work during 21st century"), + header: block + .header + .zcash_serialize_to_vec() + .expect("verified block should serialize"), + vtx: block + .transactions + .iter() + .cloned() + .enumerate() + .map(transaction_to_compact) + .collect(), + chain_metadata: Some(chain_metadata), + + ..Default::default() + } +} + +fn transaction_to_compact((index, tx): (usize, Arc)) -> CompactTx { + CompactTx { + index: index + .try_into() + .expect("tx index in block should fit in u64"), + hash: tx.hash().bytes_in_display_order().to_vec(), + + // `fee` is not checked by the `scan_block` function. + fee: 0, + + spends: tx + .sapling_nullifiers() + .map(|nf| CompactSaplingSpend { + nf: <[u8; 32]>::from(*nf).to_vec(), + }) + .collect(), + outputs: tx + .sapling_outputs() + .map(|output| CompactSaplingOutput { + cmu: output.cm_u.to_bytes().to_vec(), + ephemeral_key: output + .ephemeral_key + .zcash_serialize_to_vec() + .expect("verified output should serialize successfully"), + ciphertext: output + .enc_ciphertext + .zcash_serialize_to_vec() + .expect("verified output should serialize successfully") + .into_iter() + .take(52) + .collect(), + }) + .collect(), + + // `actions` is not checked by the `scan_block` function. + actions: vec![], + } +} + // This is a copy of zcash_primitives `fake_compact_block` where the `value` argument was changed to // be a number for easier conversion: // https://github.com/zcash/librustzcash/blob/zcash_primitives-0.13.0/zcash_client_backend/src/scanning.rs#L635 From de32a43448760b00ff98994711b4d170ba5d1fc1 Mon Sep 17 00:00:00 2001 From: arya2 Date: Tue, 17 Oct 2023 20:34:08 -0400 Subject: [PATCH 06/36] scans blocks and counts transactions --- zebra-scanner/src/tests.rs | 49 ++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/zebra-scanner/src/tests.rs b/zebra-scanner/src/tests.rs index 8e6f2c22c48..816dcc44da2 100644 --- a/zebra-scanner/src/tests.rs +++ b/zebra-scanner/src/tests.rs @@ -38,18 +38,21 @@ use zebra_chain::{ }; #[tokio::test] +#[allow(clippy::print_stdout)] async fn scanning_from_zebra() -> Result<()> { - let (consensus_config, state_config, network) = Default::default(); - - let (_, max_checkpoint_height) = - zebra_consensus::router::init_checkpoint_list(consensus_config, network); + let account = AccountId::from(12); + let extsk = ExtendedSpendingKey::master(&[]); + let dfvk: DiversifiableFullViewingKey = extsk.to_diversifiable_full_viewing_key(); + let vks: Vec<(&AccountId, &SaplingIvk)> = vec![]; + let nf = Nullifier([7; 32]); + let (state_config, network) = Default::default(); let (_state_service, read_only_state_service, _latest_chain_tip, _chain_tip_change) = - zebra_state::spawn_init(state_config, network, max_checkpoint_height, 3000).await?; - + zebra_state::spawn_init(state_config, network, Height(0), 3000).await?; let db = read_only_state_service.db(); - let mut height = Height(0); + let mut height = Height(0); + let mut num_transactions = 0; while let Some(block) = db.block(height.into()) { // Get chain metadata let sapling_tree_size = db @@ -70,20 +73,22 @@ async fn scanning_from_zebra() -> Result<()> { .expect("position should fit in u32"), }; - let _compact_block = block_to_compact(block, chain_metadata); + let compact_block = block_to_compact(block, chain_metadata); - // TODO: scan block and update state - - // let res = scan_block(&network, cb, &vks[..], &[(account, nf)], None).unwrap(); + let res = scan_block( + &zcash_primitives::consensus::MainNetwork, + compact_block, + &vks[..], + &[(account, nf)], + None, + ) + .unwrap(); + num_transactions += res.transactions().len(); height = height.next()?; } - let account = AccountId::from(12); - let extsk = ExtendedSpendingKey::master(&[]); - let dfvk = extsk.to_diversifiable_full_viewing_key(); - - let nf = Nullifier([7; 32]); + println!("num_transactions: {num_transactions}"); let cb = fake_compact_block( 1u32.into(), @@ -98,10 +103,14 @@ async fn scanning_from_zebra() -> Result<()> { // The fake block function will have our transaction and a random one. assert_eq!(cb.vtx.len(), 2); - let vks: Vec<(&AccountId, &SaplingIvk)> = vec![]; - let network = zcash_primitives::consensus::TestNetwork; - - let res = scan_block(&network, cb, &vks[..], &[(account, nf)], None).unwrap(); + let res = scan_block( + &zcash_primitives::consensus::MainNetwork, + cb, + &vks[..], + &[(account, nf)], + None, + ) + .unwrap(); // The response should have one transaction relevant to the key we provided. assert_eq!(res.transactions().len(), 1); From 0d0117d18fe723488384a655743d6afbe7a023ae Mon Sep 17 00:00:00 2001 From: arya2 Date: Wed, 18 Oct 2023 14:18:19 -0400 Subject: [PATCH 07/36] fix bug --- zebra-scanner/src/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zebra-scanner/src/tests.rs b/zebra-scanner/src/tests.rs index 816dcc44da2..1cb3ced6444 100644 --- a/zebra-scanner/src/tests.rs +++ b/zebra-scanner/src/tests.rs @@ -48,7 +48,7 @@ async fn scanning_from_zebra() -> Result<()> { let (state_config, network) = Default::default(); let (_state_service, read_only_state_service, _latest_chain_tip, _chain_tip_change) = - zebra_state::spawn_init(state_config, network, Height(0), 3000).await?; + zebra_state::spawn_init(state_config, network, Height::MAX, 3000).await?; let db = read_only_state_service.db(); let mut height = Height(0); From cee1db69864ff1b820893b868008d0b0ff0a4155 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Thu, 19 Oct 2023 09:13:40 -0300 Subject: [PATCH 08/36] split into 2 tests --- Cargo.lock | 1 + zebra-scanner/Cargo.toml | 2 + zebra-scanner/src/tests.rs | 96 +++++++++++++++++++++++++++++++------- 3 files changed, 81 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 71fc57f0d8a..68897a07e4c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5813,6 +5813,7 @@ dependencies = [ "zebra-chain", "zebra-consensus", "zebra-state", + "zebra-test", ] [[package]] diff --git a/zebra-scanner/Cargo.toml b/zebra-scanner/Cargo.toml index e7c026c07c4..e61db9bae49 100644 --- a/zebra-scanner/Cargo.toml +++ b/zebra-scanner/Cargo.toml @@ -37,6 +37,8 @@ zebra-consensus = { path = "../zebra-consensus", version = "1.0.0-beta.30" } zebra-chain = { path = "../zebra-chain", version = "1.0.0-beta.30", features = ["async-error"] } +zebra-test = { path = "../zebra-test", version = "1.0.0-beta.30" } + [dev-dependencies] color-eyre = { version = "0.6.2" } diff --git a/zebra-scanner/src/tests.rs b/zebra-scanner/src/tests.rs index 1cb3ced6444..b612c6c2061 100644 --- a/zebra-scanner/src/tests.rs +++ b/zebra-scanner/src/tests.rs @@ -32,33 +32,43 @@ use rand::{rngs::OsRng, RngCore}; use ff::{Field, PrimeField}; use group::GroupEncoding; use zebra_chain::{ - block::{Block, Height}, - serialization::ZcashSerialize, + block::Block, + chain_tip::ChainTip, + serialization::{ZcashDeserializeInto, ZcashSerialize}, transaction::Transaction, }; #[tokio::test] -#[allow(clippy::print_stdout)] -async fn scanning_from_zebra() -> Result<()> { +async fn scanning_from_populated_zebra_state() -> Result<()> { let account = AccountId::from(12); let extsk = ExtendedSpendingKey::master(&[]); let dfvk: DiversifiableFullViewingKey = extsk.to_diversifiable_full_viewing_key(); let vks: Vec<(&AccountId, &SaplingIvk)> = vec![]; let nf = Nullifier([7; 32]); - let (state_config, network) = Default::default(); - let (_state_service, read_only_state_service, _latest_chain_tip, _chain_tip_change) = - zebra_state::spawn_init(state_config, network, Height::MAX, 3000).await?; + let network = zebra_chain::parameters::Network::default(); + + // Create a continuous chain of mainnet blocks from genesis + let blocks: Vec> = zebra_test::vectors::CONTINUOUS_MAINNET_BLOCKS + .iter() + .map(|(_height, block_bytes)| block_bytes.zcash_deserialize_into().unwrap()) + .collect(); + + // Create a populated state service. + let (_state_service, read_only_state_service, latest_chain_tip, _chain_tip_change) = + zebra_state::populated_state(blocks.clone(), network).await; + let db = read_only_state_service.db(); - let mut height = Height(0); - let mut num_transactions = 0; + // use the tip as starting height + let mut height = latest_chain_tip.best_tip_height().unwrap(); + + let mut transactions_found = 0; + let mut transactions_scanned = 0; + let mut blocks_scanned = 0; while let Some(block) = db.block(height.into()) { - // Get chain metadata - let sapling_tree_size = db - .sapling_tree_by_hash_or_height(height.into()) - .expect("should exist") - .count(); + // We fake the sapling tree size to 1 because we are not in Sapling heights. + let sapling_tree_size = 1; let orchard_tree_size = db .orchard_tree_by_hash_or_height(height.into()) .expect("should exist") @@ -77,18 +87,66 @@ async fn scanning_from_zebra() -> Result<()> { let res = scan_block( &zcash_primitives::consensus::MainNetwork, - compact_block, + compact_block.clone(), &vks[..], &[(account, nf)], None, ) .unwrap(); - num_transactions += res.transactions().len(); - height = height.next()?; + transactions_found += res.transactions().len(); + transactions_scanned += compact_block.vtx.len(); + blocks_scanned += 1; + + // scan backwards + if height.is_min() { + break; + } + height = height.previous()?; } - println!("num_transactions: {num_transactions}"); + // make sure all blocks and transactions were scanned + assert_eq!(blocks_scanned, 11); + assert_eq!(transactions_scanned, 11); + + // no relevant transactions should be found + assert_eq!(transactions_found, 0); + + let cb = fake_compact_block( + 1u32.into(), + BlockHash([0; 32]), + nf, + &dfvk, + 1, + false, + Some(0), + ); + + // The fake block function will have our transaction and a random one. + assert_eq!(cb.vtx.len(), 2); + + let res = scan_block( + &zcash_primitives::consensus::MainNetwork, + cb, + &vks[..], + &[(account, nf)], + None, + ) + .unwrap(); + + // The response should have one transaction relevant to the key we provided. + assert_eq!(res.transactions().len(), 1); + + Ok(()) +} + +#[tokio::test] +async fn scanning_from_fake_generated_blocks() -> Result<()> { + let account = AccountId::from(12); + let extsk = ExtendedSpendingKey::master(&[]); + let dfvk: DiversifiableFullViewingKey = extsk.to_diversifiable_full_viewing_key(); + let vks: Vec<(&AccountId, &SaplingIvk)> = vec![]; + let nf = Nullifier([7; 32]); let cb = fake_compact_block( 1u32.into(), @@ -118,6 +176,7 @@ async fn scanning_from_zebra() -> Result<()> { Ok(()) } +// Convert a zebra block and meta data into a compact block. fn block_to_compact(block: Arc, chain_metadata: ChainMetadata) -> CompactBlock { CompactBlock { height: block @@ -154,6 +213,7 @@ fn block_to_compact(block: Arc, chain_metadata: ChainMetadata) -> Compact } } +// Convert a zebra transaction into a compact transaction. fn transaction_to_compact((index, tx): (usize, Arc)) -> CompactTx { CompactTx { index: index From 83e0542e66353a84b277ddb2c6091a80bf9b33b8 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Thu, 19 Oct 2023 14:17:21 -0300 Subject: [PATCH 09/36] add duplicated dependencies to deny.toml --- deny.toml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/deny.toml b/deny.toml index fcb321d3a67..b62e053ec8b 100644 --- a/deny.toml +++ b/deny.toml @@ -41,6 +41,14 @@ deny = [ # # Certain crates/versions that will be skipped when doing duplicate detection. skip = [ + # wait for zebra to update `incrementalmerkletree` ECC dependency + { name = "incrementalmerkletree", version = "=0.4.0" }, + + # wait for zebra to update `orchard` ECC dependency + { name = "orchard", version = "=0.5.0" }, + + # wait for zebra to update `zcash_primitives` ECC dependency + { name = "zcash_primitives", version = "=0.12.0" }, ] # Similarly to `skip` allows you to skip certain crates during duplicate From 69135257c11865b197f55efa57bff0529f210b6e Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Fri, 20 Oct 2023 14:57:26 -0300 Subject: [PATCH 10/36] upgrade zebra-scanner version --- Cargo.lock | 14 +++++++------- zebra-scanner/Cargo.toml | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 68897a07e4c..46a78cb60bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5412,9 +5412,9 @@ dependencies = [ [[package]] name = "zcash_client_backend" -version = "0.10.0" +version = "0.10.0-rc.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6a382af39be9ee5a3788157145c404b7cd19acc440903f6c34b09fb44f0e991" +checksum = "ecc33f71747a93d509f7e1c047961e359a271bdf4869cc07f7f65ee1ba7df8c2" dependencies = [ "base64 0.21.4", "bech32", @@ -5440,7 +5440,7 @@ dependencies = [ "zcash_address", "zcash_encoding", "zcash_note_encryption", - "zcash_primitives 0.13.0", + "zcash_primitives 0.13.0-rc.1", ] [[package]] @@ -5515,9 +5515,9 @@ dependencies = [ [[package]] name = "zcash_primitives" -version = "0.13.0" +version = "0.13.0-rc.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d17e4c94ca8d69d2fcf2be97522da5732a580eb2125cda3b150761952f8df8e6" +checksum = "0cc4391d9325e0a51a7cbff02b5c4b5472d66087bd9c903ddb12dea7ec22f3e0" dependencies = [ "aes", "bip0039", @@ -5798,7 +5798,7 @@ dependencies = [ [[package]] name = "zebra-scanner" -version = "1.0.0-beta.29" +version = "1.0.0-beta.30" dependencies = [ "bls12_381", "color-eyre", @@ -5809,7 +5809,7 @@ dependencies = [ "tokio", "zcash_client_backend", "zcash_note_encryption", - "zcash_primitives 0.13.0", + "zcash_primitives 0.13.0-rc.1", "zebra-chain", "zebra-consensus", "zebra-state", diff --git a/zebra-scanner/Cargo.toml b/zebra-scanner/Cargo.toml index e61db9bae49..21cdcf2632b 100644 --- a/zebra-scanner/Cargo.toml +++ b/zebra-scanner/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zebra-scanner" -version = "1.0.0-beta.29" +version = "1.0.0-beta.30" authors = ["Zcash Foundation "] description = "Blockchain scanner for the Zebra Zcash node" license = "MIT OR Apache-2.0" @@ -19,8 +19,8 @@ categories = ["cryptography::cryptocurrencies"] # Production features that activate extra dependencies, or extra features in dependencies [dependencies] -zcash_client_backend = "0.10.0" -zcash_primitives = "0.13.0" +zcash_client_backend = "0.10.0-rc.1" +zcash_primitives = "=0.13.0-rc.1" zcash_note_encryption = "0.4.0" rand = "0.8.5" From 8f1e6e1b17b31e2f97f04c9896fc865960b84ddc Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Fri, 20 Oct 2023 16:33:58 -0300 Subject: [PATCH 11/36] try removing ecc duplicated dependencies --- deny.toml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/deny.toml b/deny.toml index b62e053ec8b..fcb321d3a67 100644 --- a/deny.toml +++ b/deny.toml @@ -41,14 +41,6 @@ deny = [ # # Certain crates/versions that will be skipped when doing duplicate detection. skip = [ - # wait for zebra to update `incrementalmerkletree` ECC dependency - { name = "incrementalmerkletree", version = "=0.4.0" }, - - # wait for zebra to update `orchard` ECC dependency - { name = "orchard", version = "=0.5.0" }, - - # wait for zebra to update `zcash_primitives` ECC dependency - { name = "zcash_primitives", version = "=0.12.0" }, ] # Similarly to `skip` allows you to skip certain crates during duplicate From b88e932829802540176b471ff88a0f5949ecd407 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Fri, 20 Oct 2023 17:16:14 -0300 Subject: [PATCH 12/36] try fix deny.toml --- deny.toml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/deny.toml b/deny.toml index fcb321d3a67..ab42a18980a 100644 --- a/deny.toml +++ b/deny.toml @@ -10,7 +10,7 @@ [bans] # Lint level for when multiple versions of the same crate are detected multiple-versions = "deny" - +tails # Don't allow wildcard ("any version") dependencies wildcards = "deny" # Allow private and dev wildcard dependencies. @@ -72,13 +72,10 @@ skip-tree = [ # wait for indexmap, toml_edit, serde_json, tower to upgrade { name = "hashbrown", version = "=0.12.3" }, - # ECC crates - - # wait for minreq and zcash_proofs to upgrade - { name = "rustls", version = "=0.20.9" }, + # wait for jsonrpc-http-server to upgrade + { name = "redox_syscall", version = "=0.2.16" }, - # wait for zcash_proofs to upgrade - { name = "webpki-roots", version = "=0.22.6" }, + # ECC crates # zebra-utils dependencies From 598cda65bbd36f4a1629acb948b49770f24d3b71 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Fri, 20 Oct 2023 17:32:58 -0300 Subject: [PATCH 13/36] remove unintended paste from deny.toml --- deny.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deny.toml b/deny.toml index ab42a18980a..84425146a73 100644 --- a/deny.toml +++ b/deny.toml @@ -10,7 +10,7 @@ [bans] # Lint level for when multiple versions of the same crate are detected multiple-versions = "deny" -tails + # Don't allow wildcard ("any version") dependencies wildcards = "deny" # Allow private and dev wildcard dependencies. From 19dfaa7d47dd1dae3121125304948f6cfff5ef4d Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Tue, 24 Oct 2023 11:40:33 -0300 Subject: [PATCH 14/36] remove duplicated code from the other test --- zebra-scanner/src/tests.rs | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/zebra-scanner/src/tests.rs b/zebra-scanner/src/tests.rs index b612c6c2061..cfa867d5dc0 100644 --- a/zebra-scanner/src/tests.rs +++ b/zebra-scanner/src/tests.rs @@ -41,8 +41,6 @@ use zebra_chain::{ #[tokio::test] async fn scanning_from_populated_zebra_state() -> Result<()> { let account = AccountId::from(12); - let extsk = ExtendedSpendingKey::master(&[]); - let dfvk: DiversifiableFullViewingKey = extsk.to_diversifiable_full_viewing_key(); let vks: Vec<(&AccountId, &SaplingIvk)> = vec![]; let nf = Nullifier([7; 32]); @@ -112,31 +110,6 @@ async fn scanning_from_populated_zebra_state() -> Result<()> { // no relevant transactions should be found assert_eq!(transactions_found, 0); - let cb = fake_compact_block( - 1u32.into(), - BlockHash([0; 32]), - nf, - &dfvk, - 1, - false, - Some(0), - ); - - // The fake block function will have our transaction and a random one. - assert_eq!(cb.vtx.len(), 2); - - let res = scan_block( - &zcash_primitives::consensus::MainNetwork, - cb, - &vks[..], - &[(account, nf)], - None, - ) - .unwrap(); - - // The response should have one transaction relevant to the key we provided. - assert_eq!(res.transactions().len(), 1); - Ok(()) } From 2ca1c93ce72456d06e0d2ca26ed7bcff0f946522 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Wed, 25 Oct 2023 13:10:46 -0300 Subject: [PATCH 15/36] remove strict version of `zcash_primitives` crate --- zebra-scanner/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zebra-scanner/Cargo.toml b/zebra-scanner/Cargo.toml index 21cdcf2632b..1fe3db9ed49 100644 --- a/zebra-scanner/Cargo.toml +++ b/zebra-scanner/Cargo.toml @@ -20,7 +20,7 @@ categories = ["cryptography::cryptocurrencies"] [dependencies] zcash_client_backend = "0.10.0-rc.1" -zcash_primitives = "=0.13.0-rc.1" +zcash_primitives = "0.13.0-rc.1" zcash_note_encryption = "0.4.0" rand = "0.8.5" From 273fd95efaa182bfcf507c302a8bce4d3497b8f8 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Wed, 25 Oct 2023 14:26:43 -0300 Subject: [PATCH 16/36] change description --- zebra-scanner/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zebra-scanner/Cargo.toml b/zebra-scanner/Cargo.toml index 1fe3db9ed49..27ac98e8642 100644 --- a/zebra-scanner/Cargo.toml +++ b/zebra-scanner/Cargo.toml @@ -2,7 +2,7 @@ name = "zebra-scanner" version = "1.0.0-beta.30" authors = ["Zcash Foundation "] -description = "Blockchain scanner for the Zebra Zcash node" +description = "Shielded transaction scanner for the Zcash blockchain" license = "MIT OR Apache-2.0" repository = "https://github.com/ZcashFoundation/zebra" edition = "2021" From b2967695ba1ec6b64aad478cbeade71de56ca2e0 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Wed, 25 Oct 2023 14:39:35 -0300 Subject: [PATCH 17/36] remove feture --- zebra-scanner/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zebra-scanner/Cargo.toml b/zebra-scanner/Cargo.toml index 27ac98e8642..2b556025aa6 100644 --- a/zebra-scanner/Cargo.toml +++ b/zebra-scanner/Cargo.toml @@ -35,7 +35,7 @@ tokio = { version = "1.33.0", features = ["full", "tracing", "test-util"] } zebra-state = { path = "../zebra-state", version = "1.0.0-beta.30" } zebra-consensus = { path = "../zebra-consensus", version = "1.0.0-beta.30" } -zebra-chain = { path = "../zebra-chain", version = "1.0.0-beta.30", features = ["async-error"] } +zebra-chain = { path = "../zebra-chain", version = "1.0.0-beta.30" } zebra-test = { path = "../zebra-test", version = "1.0.0-beta.30" } From 51affbab5e9eeb8df41ec6a15f467ee721b9e271 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Wed, 25 Oct 2023 14:49:44 -0300 Subject: [PATCH 18/36] remove tokio features --- zebra-scanner/Cargo.toml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/zebra-scanner/Cargo.toml b/zebra-scanner/Cargo.toml index 2b556025aa6..acc12b66df4 100644 --- a/zebra-scanner/Cargo.toml +++ b/zebra-scanner/Cargo.toml @@ -24,19 +24,15 @@ zcash_primitives = "0.13.0-rc.1" zcash_note_encryption = "0.4.0" rand = "0.8.5" - bls12_381 = "0.8.0" jubjub = "0.10.0" ff = "0.13.0" group = "0.13.0" - -tokio = { version = "1.33.0", features = ["full", "tracing", "test-util"] } +tokio = { version = "1.33.0", features = ["test-util"] } zebra-state = { path = "../zebra-state", version = "1.0.0-beta.30" } zebra-consensus = { path = "../zebra-consensus", version = "1.0.0-beta.30" } - zebra-chain = { path = "../zebra-chain", version = "1.0.0-beta.30" } - zebra-test = { path = "../zebra-test", version = "1.0.0-beta.30" } [dev-dependencies] From 56218360611d32d1b6aedd57f9639ccf4eb4409e Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Wed, 25 Oct 2023 14:52:58 -0300 Subject: [PATCH 19/36] change lib doc --- zebra-scanner/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zebra-scanner/src/lib.rs b/zebra-scanner/src/lib.rs index ec988b34dae..0feff7cab5a 100644 --- a/zebra-scanner/src/lib.rs +++ b/zebra-scanner/src/lib.rs @@ -1,4 +1,4 @@ -//! Blockchain scanner code for Zebra. 🦓 +//! Shielded transaction scanner for the Zcash blockchain. #![doc(html_favicon_url = "https://zfnd.org/wp-content/uploads/2022/03/zebra-favicon-128.png")] #![doc(html_logo_url = "https://zfnd.org/wp-content/uploads/2022/03/zebra-icon.png")] From b455a371c7e0dd3d0927cc1c7a3df4e56d38c71f Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Wed, 25 Oct 2023 15:31:14 -0300 Subject: [PATCH 20/36] add more documentation --- zebra-scanner/src/tests.rs | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/zebra-scanner/src/tests.rs b/zebra-scanner/src/tests.rs index cfa867d5dc0..62b54558a76 100644 --- a/zebra-scanner/src/tests.rs +++ b/zebra-scanner/src/tests.rs @@ -1,5 +1,7 @@ -//! Test that we can scan blocks. +//! Test that we can scan the Zebra blockchain using the external `zcash_client_backend` crate +//! scanning functionality. //! +//! This tests belong to the proof of concept stage of the external wallet support functionality. use std::sync::Arc; @@ -38,6 +40,10 @@ use zebra_chain::{ transaction::Transaction, }; +/// Prove that Zebra blocks can be scanned using the `zcash_client_backend::scanning::scan_block` function: +/// - Populates the state with a continuous chain of mainnet blocks from genesis. +/// - Scan the chain from the tip going backwards down to genesis. +/// - Verifies that no relevant transaction is found in the chain when scanning for a fake account's nullifier. #[tokio::test] async fn scanning_from_populated_zebra_state() -> Result<()> { let account = AccountId::from(12); @@ -113,6 +119,12 @@ async fn scanning_from_populated_zebra_state() -> Result<()> { Ok(()) } +/// Prove that we can create fake blocks with fake notes and scan them using the +/// `zcash_client_backend::scanning::scan_block` function: +/// - Function `fake_compact_block` will generate 1 block with one pre created fake nullifier in +/// the transaction and one additional random transaction without it. +/// - Verify one relevant transaction is found in the chain when scanning for the pre created fake +/// account's nullifier. #[tokio::test] async fn scanning_from_fake_generated_blocks() -> Result<()> { let account = AccountId::from(12); @@ -149,7 +161,7 @@ async fn scanning_from_fake_generated_blocks() -> Result<()> { Ok(()) } -// Convert a zebra block and meta data into a compact block. +/// Convert a zebra block and meta data into a compact block. fn block_to_compact(block: Arc, chain_metadata: ChainMetadata) -> CompactBlock { CompactBlock { height: block @@ -186,7 +198,7 @@ fn block_to_compact(block: Arc, chain_metadata: ChainMetadata) -> Compact } } -// Convert a zebra transaction into a compact transaction. +/// Convert a zebra transaction into a compact transaction. fn transaction_to_compact((index, tx): (usize, Arc)) -> CompactTx { CompactTx { index: index @@ -226,6 +238,7 @@ fn transaction_to_compact((index, tx): (usize, Arc)) -> CompactTx { } } +/// Create a fake compact block with provided fake account data. // This is a copy of zcash_primitives `fake_compact_block` where the `value` argument was changed to // be a number for easier conversion: // https://github.com/zcash/librustzcash/blob/zcash_primitives-0.13.0/zcash_client_backend/src/scanning.rs#L635 @@ -310,6 +323,7 @@ fn fake_compact_block( cb } +/// Create a random compact transaction. // This is an exact copy of `zcash_client_backend::scanning::random_compact_tx`: // https://github.com/zcash/librustzcash/blob/zcash_primitives-0.13.0/zcash_client_backend/src/scanning.rs#L597 // We need to copy because this is a test private function upstream. From 475a771c16151d00133956e92ac3a6cc219411e0 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Wed, 25 Oct 2023 16:16:27 -0300 Subject: [PATCH 21/36] change expect --- zebra-scanner/src/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zebra-scanner/src/tests.rs b/zebra-scanner/src/tests.rs index 62b54558a76..321a269772d 100644 --- a/zebra-scanner/src/tests.rs +++ b/zebra-scanner/src/tests.rs @@ -180,7 +180,7 @@ fn block_to_compact(block: Arc, chain_metadata: ChainMetadata) -> Compact .time .timestamp() .try_into() - .expect("should work during 21st century"), + .expect("unsigned 32-bit times should work until 2105"), header: block .header .zcash_serialize_to_vec() From 3909795ac7ab5a85f2ddc8f249ec574d8f7b9793 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Wed, 25 Oct 2023 16:31:21 -0300 Subject: [PATCH 22/36] do not use default in compact block creation --- zebra-scanner/src/tests.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zebra-scanner/src/tests.rs b/zebra-scanner/src/tests.rs index 321a269772d..495507b4d9b 100644 --- a/zebra-scanner/src/tests.rs +++ b/zebra-scanner/src/tests.rs @@ -194,7 +194,8 @@ fn block_to_compact(block: Arc, chain_metadata: ChainMetadata) -> Compact .collect(), chain_metadata: Some(chain_metadata), - ..Default::default() + // The protocol version is used for the gRPC wire format, so it isn't needed here. + proto_version: 0, } } From 25e205bf0771451a6a2087fe4716cf68fb548bf9 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Wed, 25 Oct 2023 16:48:17 -0300 Subject: [PATCH 23/36] more docs --- zebra-scanner/src/tests.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/zebra-scanner/src/tests.rs b/zebra-scanner/src/tests.rs index 495507b4d9b..3dfc66d2e10 100644 --- a/zebra-scanner/src/tests.rs +++ b/zebra-scanner/src/tests.rs @@ -207,7 +207,8 @@ fn transaction_to_compact((index, tx): (usize, Arc)) -> CompactTx { .expect("tx index in block should fit in u64"), hash: tx.hash().bytes_in_display_order().to_vec(), - // `fee` is not checked by the `scan_block` function. + // `fee` is not checked by the `scan_block` function. It is allowed to be unset. + // fee: 0, spends: tx @@ -216,6 +217,10 @@ fn transaction_to_compact((index, tx): (usize, Arc)) -> CompactTx { nf: <[u8; 32]>::from(*nf).to_vec(), }) .collect(), + + // > output encodes the cmu field, ephemeralKey field, and a 52-byte prefix of the encCiphertext field of a Sapling Output + // + // outputs: tx .sapling_outputs() .map(|output| CompactSaplingOutput { From 550681e7c83d1e230a1c40f85da86ea9c43c0f1e Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Wed, 25 Oct 2023 17:02:54 -0300 Subject: [PATCH 24/36] add more checks to test --- zebra-scanner/src/tests.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/zebra-scanner/src/tests.rs b/zebra-scanner/src/tests.rs index 3dfc66d2e10..fd6c2df1195 100644 --- a/zebra-scanner/src/tests.rs +++ b/zebra-scanner/src/tests.rs @@ -148,7 +148,7 @@ async fn scanning_from_fake_generated_blocks() -> Result<()> { let res = scan_block( &zcash_primitives::consensus::MainNetwork, - cb, + cb.clone(), &vks[..], &[(account, nf)], None, @@ -157,6 +157,11 @@ async fn scanning_from_fake_generated_blocks() -> Result<()> { // The response should have one transaction relevant to the key we provided. assert_eq!(res.transactions().len(), 1); + // The transaction should be the one we provided, second one in the block. + // (random transaction is added before ours in `fake_compact_block` function) + assert_eq!(res.transactions()[0].txid, cb.vtx[1].txid()); + // The block hash of the response should be the same as the one provided. + assert_eq!(res.block_hash(), cb.hash()); Ok(()) } From 6c83618d6e7dd341d1deb419000a6ba855363590 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Thu, 26 Oct 2023 14:52:01 -0300 Subject: [PATCH 25/36] remove zebra-consensus dependency --- Cargo.lock | 1 - zebra-scanner/Cargo.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 28298b85103..9a382a05278 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5744,7 +5744,6 @@ dependencies = [ "zcash_note_encryption", "zcash_primitives", "zebra-chain", - "zebra-consensus", "zebra-state", "zebra-test", ] diff --git a/zebra-scanner/Cargo.toml b/zebra-scanner/Cargo.toml index acc12b66df4..a42463067c6 100644 --- a/zebra-scanner/Cargo.toml +++ b/zebra-scanner/Cargo.toml @@ -31,7 +31,6 @@ group = "0.13.0" tokio = { version = "1.33.0", features = ["test-util"] } zebra-state = { path = "../zebra-state", version = "1.0.0-beta.30" } -zebra-consensus = { path = "../zebra-consensus", version = "1.0.0-beta.30" } zebra-chain = { path = "../zebra-chain", version = "1.0.0-beta.30" } zebra-test = { path = "../zebra-test", version = "1.0.0-beta.30" } From ad19e00a7bf8a497e8b6fb1386146fe0d3fbc5f0 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Thu, 26 Oct 2023 15:11:20 -0300 Subject: [PATCH 26/36] move all deps to dev-deps --- zebra-scanner/Cargo.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/zebra-scanner/Cargo.toml b/zebra-scanner/Cargo.toml index a42463067c6..c919ce23649 100644 --- a/zebra-scanner/Cargo.toml +++ b/zebra-scanner/Cargo.toml @@ -19,10 +19,14 @@ categories = ["cryptography::cryptocurrencies"] # Production features that activate extra dependencies, or extra features in dependencies [dependencies] + +[dev-dependencies] + zcash_client_backend = "0.10.0-rc.1" zcash_primitives = "0.13.0-rc.1" zcash_note_encryption = "0.4.0" +color-eyre = { version = "0.6.2" } rand = "0.8.5" bls12_381 = "0.8.0" jubjub = "0.10.0" @@ -33,7 +37,3 @@ tokio = { version = "1.33.0", features = ["test-util"] } zebra-state = { path = "../zebra-state", version = "1.0.0-beta.30" } zebra-chain = { path = "../zebra-chain", version = "1.0.0-beta.30" } zebra-test = { path = "../zebra-test", version = "1.0.0-beta.30" } - -[dev-dependencies] - -color-eyre = { version = "0.6.2" } From d10940ec42e1f4d30174a561b3d1202c51407376 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Thu, 26 Oct 2023 15:18:08 -0300 Subject: [PATCH 27/36] change crate version --- Cargo.lock | 2 +- zebra-scanner/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9a382a05278..707cb454ade 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5731,7 +5731,7 @@ dependencies = [ [[package]] name = "zebra-scanner" -version = "1.0.0-beta.30" +version = "0.1.0-alpha.0" dependencies = [ "bls12_381", "color-eyre", diff --git a/zebra-scanner/Cargo.toml b/zebra-scanner/Cargo.toml index c919ce23649..3f7e03ab333 100644 --- a/zebra-scanner/Cargo.toml +++ b/zebra-scanner/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zebra-scanner" -version = "1.0.0-beta.30" +version = "0.1.0-alpha.0" authors = ["Zcash Foundation "] description = "Shielded transaction scanner for the Zcash blockchain" license = "MIT OR Apache-2.0" From d8bd9ae0aca995777d6f79d28e2378962b2ec88b Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Wed, 1 Nov 2023 12:16:42 -0300 Subject: [PATCH 28/36] rename crate to zebra-scan --- Cargo.lock | 2 +- Cargo.toml | 2 +- {zebra-scanner => zebra-scan}/Cargo.toml | 2 +- {zebra-scanner => zebra-scan}/src/lib.rs | 0 {zebra-scanner => zebra-scan}/src/tests.rs | 0 5 files changed, 3 insertions(+), 3 deletions(-) rename {zebra-scanner => zebra-scan}/Cargo.toml (97%) rename {zebra-scanner => zebra-scan}/src/lib.rs (100%) rename {zebra-scanner => zebra-scan}/src/tests.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index 9e7b32667d7..5ed696732fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5767,7 +5767,7 @@ dependencies = [ ] [[package]] -name = "zebra-scanner" +name = "zebra-scan" version = "0.1.0-alpha.0" dependencies = [ "bls12_381", diff --git a/Cargo.toml b/Cargo.toml index 3ccdf382cd4..4c8b5f651e1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ members = [ "zebra-node-services", "zebra-test", "zebra-utils", - "zebra-scanner", + "zebra-scan", "tower-batch-control", "tower-fallback", ] diff --git a/zebra-scanner/Cargo.toml b/zebra-scan/Cargo.toml similarity index 97% rename from zebra-scanner/Cargo.toml rename to zebra-scan/Cargo.toml index 3f7e03ab333..469f132a452 100644 --- a/zebra-scanner/Cargo.toml +++ b/zebra-scan/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "zebra-scanner" +name = "zebra-scan" version = "0.1.0-alpha.0" authors = ["Zcash Foundation "] description = "Shielded transaction scanner for the Zcash blockchain" diff --git a/zebra-scanner/src/lib.rs b/zebra-scan/src/lib.rs similarity index 100% rename from zebra-scanner/src/lib.rs rename to zebra-scan/src/lib.rs diff --git a/zebra-scanner/src/tests.rs b/zebra-scan/src/tests.rs similarity index 100% rename from zebra-scanner/src/tests.rs rename to zebra-scan/src/tests.rs From 9b08dea9782efd1f2aea48682aac76a51c9e38b3 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Wed, 1 Nov 2023 12:56:27 -0300 Subject: [PATCH 29/36] lock file --- Cargo.lock | 9 --------- 1 file changed, 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 309d2460232..2d5cd0991d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3427,15 +3427,6 @@ dependencies = [ "bitflags 1.3.2", ] -[[package]] -name = "redox_syscall" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" -dependencies = [ - "bitflags 1.3.2", -] - [[package]] name = "redox_syscall" version = "0.4.1" From 073495ed4c5abb9b2e49fabee54d9927cdacc0aa Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Mon, 6 Nov 2023 16:23:54 -0300 Subject: [PATCH 30/36] add test for zecpages populated state --- zebra-scan/src/tests.rs | 95 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 94 insertions(+), 1 deletion(-) diff --git a/zebra-scan/src/tests.rs b/zebra-scan/src/tests.rs index fd6c2df1195..83ec97c61b8 100644 --- a/zebra-scan/src/tests.rs +++ b/zebra-scan/src/tests.rs @@ -6,6 +6,7 @@ use std::sync::Arc; use zcash_client_backend::{ + encoding::decode_extended_full_viewing_key, proto::compact_formats::{ self as compact, ChainMetadata, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx, @@ -16,7 +17,7 @@ use zcash_note_encryption::Domain; use zcash_primitives::{ block::BlockHash, consensus::{BlockHeight, Network}, - constants::SPENDING_KEY_GENERATOR, + constants::{mainnet::HRP_SAPLING_EXTENDED_FULL_VIEWING_KEY, SPENDING_KEY_GENERATOR}, memo::MemoBytes, sapling::{ note_encryption::{sapling_note_encryption, SaplingDomain}, @@ -166,6 +167,98 @@ async fn scanning_from_fake_generated_blocks() -> Result<()> { Ok(()) } +/// Scan a populated state for the ZECpages viewing key. +/// This test is very similar to `scanning_from_populated_zebra_state` but with the ZECpages key. +/// There are no zechub transactions in the test data so we should get empty related transactions. +#[tokio::test] +async fn scanning_zecpages_from_populated_zebra_state() -> Result<()> { + /// The extended Sapling viewing key of [ZECpages](https://zecpages.com/boardinfo) + const ZECPAGES_VIEWING_KEY: &str = "zxviews1q0duytgcqqqqpqre26wkl45gvwwwd706xw608hucmvfalr759ejwf7qshjf5r9aa7323zulvz6plhttp5mltqcgs9t039cx2d09mgq05ts63n8u35hyv6h9nc9ctqqtue2u7cer2mqegunuulq2luhq3ywjcz35yyljewa4mgkgjzyfwh6fr6jd0dzd44ghk0nxdv2hnv4j5nxfwv24rwdmgllhe0p8568sgqt9ckt02v2kxf5ahtql6s0ltjpkckw8gtymxtxuu9gcr0swvz"; + + // Parse the key from ZECpages + let efvk = decode_extended_full_viewing_key( + HRP_SAPLING_EXTENDED_FULL_VIEWING_KEY, + ZECPAGES_VIEWING_KEY, + ) + .unwrap(); + + let account = AccountId::from(1); + let nf = Nullifier([7; 32]); + + // Build a vector of viewing keys `vks` to scan for. + let fvk = efvk.fvk; + let ivk = fvk.vk.ivk(); + let vks: Vec<(&AccountId, &SaplingIvk)> = vec![(&account, &ivk)]; + + let network = zebra_chain::parameters::Network::default(); + + // Create a continuous chain of mainnet blocks from genesis + let blocks: Vec> = zebra_test::vectors::CONTINUOUS_MAINNET_BLOCKS + .iter() + .map(|(_height, block_bytes)| block_bytes.zcash_deserialize_into().unwrap()) + .collect(); + + // Create a populated state service. + let (_state_service, read_only_state_service, latest_chain_tip, _chain_tip_change) = + zebra_state::populated_state(blocks.clone(), network).await; + + let db = read_only_state_service.db(); + + // use the tip as starting height + let mut height = latest_chain_tip.best_tip_height().unwrap(); + + let mut transactions_found = 0; + let mut transactions_scanned = 0; + let mut blocks_scanned = 0; + while let Some(block) = db.block(height.into()) { + // We fake the sapling tree size to 1 because we are not in Sapling heights. + let sapling_tree_size = 1; + let orchard_tree_size = db + .orchard_tree_by_hash_or_height(height.into()) + .expect("should exist") + .count(); + + let chain_metadata = ChainMetadata { + sapling_commitment_tree_size: sapling_tree_size + .try_into() + .expect("position should fit in u32"), + orchard_commitment_tree_size: orchard_tree_size + .try_into() + .expect("position should fit in u32"), + }; + + let compact_block = block_to_compact(block, chain_metadata); + + let res = scan_block( + &zcash_primitives::consensus::MainNetwork, + compact_block.clone(), + &vks[..], + &[(account, nf)], + None, + ) + .unwrap(); + + transactions_found += res.transactions().len(); + transactions_scanned += compact_block.vtx.len(); + blocks_scanned += 1; + + // scan backwards + if height.is_min() { + break; + } + height = height.previous()?; + } + + // make sure all blocks and transactions were scanned + assert_eq!(blocks_scanned, 11); + assert_eq!(transactions_scanned, 11); + + // no relevant transactions should be found + assert_eq!(transactions_found, 0); + + Ok(()) +} + /// Convert a zebra block and meta data into a compact block. fn block_to_compact(block: Arc, chain_metadata: ChainMetadata) -> CompactBlock { CompactBlock { From 52958eb609ab01db21abc69d92a40fa28b24d8c0 Mon Sep 17 00:00:00 2001 From: Arya Date: Tue, 7 Nov 2023 13:46:59 -0500 Subject: [PATCH 31/36] scans all cached blocks for zecpages viewing key expecting Ok results. --- zebra-scan/src/tests.rs | 59 ++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/zebra-scan/src/tests.rs b/zebra-scan/src/tests.rs index 83ec97c61b8..d155eecc898 100644 --- a/zebra-scan/src/tests.rs +++ b/zebra-scan/src/tests.rs @@ -6,6 +6,7 @@ use std::sync::Arc; use zcash_client_backend::{ + data_api::BlockMetadata, encoding::decode_extended_full_viewing_key, proto::compact_formats::{ self as compact, ChainMetadata, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, @@ -183,7 +184,6 @@ async fn scanning_zecpages_from_populated_zebra_state() -> Result<()> { .unwrap(); let account = AccountId::from(1); - let nf = Nullifier([7; 32]); // Build a vector of viewing keys `vks` to scan for. let fvk = efvk.fvk; @@ -191,28 +191,30 @@ async fn scanning_zecpages_from_populated_zebra_state() -> Result<()> { let vks: Vec<(&AccountId, &SaplingIvk)> = vec![(&account, &ivk)]; let network = zebra_chain::parameters::Network::default(); + let state_config = Default::default(); - // Create a continuous chain of mainnet blocks from genesis - let blocks: Vec> = zebra_test::vectors::CONTINUOUS_MAINNET_BLOCKS - .iter() - .map(|(_height, block_bytes)| block_bytes.zcash_deserialize_into().unwrap()) - .collect(); - - // Create a populated state service. let (_state_service, read_only_state_service, latest_chain_tip, _chain_tip_change) = - zebra_state::populated_state(blocks.clone(), network).await; - + zebra_state::spawn_init(state_config, network, zebra_chain::block::Height::MAX, 3000) + .await?; let db = read_only_state_service.db(); // use the tip as starting height let mut height = latest_chain_tip.best_tip_height().unwrap(); - let mut transactions_found = 0; - let mut transactions_scanned = 0; - let mut blocks_scanned = 0; while let Some(block) = db.block(height.into()) { - // We fake the sapling tree size to 1 because we are not in Sapling heights. - let sapling_tree_size = 1; + let sapling_tree_size = db + .sapling_tree_by_hash_or_height(height.into()) + .expect("should exist") + .count(); + + let previous_sapling_tree_size = if height.is_min() { + 0 + } else { + db.sapling_tree_by_hash_or_height(block.header.previous_block_hash.into()) + .expect("should exist") + .count() + }; + let orchard_tree_size = db .orchard_tree_by_hash_or_height(height.into()) .expect("should exist") @@ -227,20 +229,24 @@ async fn scanning_zecpages_from_populated_zebra_state() -> Result<()> { .expect("position should fit in u32"), }; + let block_metadata = BlockMetadata::from_parts( + height.previous()?.0.into(), + BlockHash(block.header.previous_block_hash.0), + previous_sapling_tree_size + .try_into() + .expect("should fit in u32"), + ); + let compact_block = block_to_compact(block, chain_metadata); - let res = scan_block( + scan_block( &zcash_primitives::consensus::MainNetwork, compact_block.clone(), &vks[..], - &[(account, nf)], - None, + &[], + Some(&block_metadata), ) - .unwrap(); - - transactions_found += res.transactions().len(); - transactions_scanned += compact_block.vtx.len(); - blocks_scanned += 1; + .expect("should scan block successfully"); // scan backwards if height.is_min() { @@ -249,13 +255,6 @@ async fn scanning_zecpages_from_populated_zebra_state() -> Result<()> { height = height.previous()?; } - // make sure all blocks and transactions were scanned - assert_eq!(blocks_scanned, 11); - assert_eq!(transactions_scanned, 11); - - // no relevant transactions should be found - assert_eq!(transactions_found, 0); - Ok(()) } From 133e461807879bb1a001b357f77095a4096aeed5 Mon Sep 17 00:00:00 2001 From: Arya Date: Tue, 7 Nov 2023 14:04:18 -0500 Subject: [PATCH 32/36] use test blocks --- zebra-scan/src/tests.rs | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/zebra-scan/src/tests.rs b/zebra-scan/src/tests.rs index d155eecc898..75b337ee738 100644 --- a/zebra-scan/src/tests.rs +++ b/zebra-scan/src/tests.rs @@ -191,16 +191,25 @@ async fn scanning_zecpages_from_populated_zebra_state() -> Result<()> { let vks: Vec<(&AccountId, &SaplingIvk)> = vec![(&account, &ivk)]; let network = zebra_chain::parameters::Network::default(); - let state_config = Default::default(); + // Create a continuous chain of mainnet blocks from genesis + let blocks: Vec> = zebra_test::vectors::CONTINUOUS_MAINNET_BLOCKS + .iter() + .map(|(_height, block_bytes)| block_bytes.zcash_deserialize_into().unwrap()) + .collect(); + + // Create a populated state service. let (_state_service, read_only_state_service, latest_chain_tip, _chain_tip_change) = - zebra_state::spawn_init(state_config, network, zebra_chain::block::Height::MAX, 3000) - .await?; + zebra_state::populated_state(blocks.clone(), network).await; + let db = read_only_state_service.db(); // use the tip as starting height let mut height = latest_chain_tip.best_tip_height().unwrap(); + let mut transactions_found = 0; + let mut transactions_scanned = 0; + let mut blocks_scanned = 0; while let Some(block) = db.block(height.into()) { let sapling_tree_size = db .sapling_tree_by_hash_or_height(height.into()) @@ -239,7 +248,7 @@ async fn scanning_zecpages_from_populated_zebra_state() -> Result<()> { let compact_block = block_to_compact(block, chain_metadata); - scan_block( + let res = scan_block( &zcash_primitives::consensus::MainNetwork, compact_block.clone(), &vks[..], @@ -248,6 +257,10 @@ async fn scanning_zecpages_from_populated_zebra_state() -> Result<()> { ) .expect("should scan block successfully"); + transactions_found += res.transactions().len(); + transactions_scanned += compact_block.vtx.len(); + blocks_scanned += 1; + // scan backwards if height.is_min() { break; @@ -255,6 +268,13 @@ async fn scanning_zecpages_from_populated_zebra_state() -> Result<()> { height = height.previous()?; } + // make sure all blocks and transactions were scanned + assert_eq!(blocks_scanned, 11); + assert_eq!(transactions_scanned, 11); + + // no relevant transactions should be found + assert_eq!(transactions_found, 0); + Ok(()) } From 46a5e366206d4d97ed1f61bfaa096d1c3c842641 Mon Sep 17 00:00:00 2001 From: Arya Date: Tue, 7 Nov 2023 14:13:07 -0500 Subject: [PATCH 33/36] fixes test --- zebra-scan/src/tests.rs | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/zebra-scan/src/tests.rs b/zebra-scan/src/tests.rs index 75b337ee738..de66abaa3f5 100644 --- a/zebra-scan/src/tests.rs +++ b/zebra-scan/src/tests.rs @@ -211,15 +211,11 @@ async fn scanning_zecpages_from_populated_zebra_state() -> Result<()> { let mut transactions_scanned = 0; let mut blocks_scanned = 0; while let Some(block) = db.block(height.into()) { - let sapling_tree_size = db - .sapling_tree_by_hash_or_height(height.into()) - .expect("should exist") - .count(); - - let previous_sapling_tree_size = if height.is_min() { - 0 + // TODO: fix this issue in zcash_client_backend + let sapling_tree_size = if height.is_min() { + 1 } else { - db.sapling_tree_by_hash_or_height(block.header.previous_block_hash.into()) + db.sapling_tree_by_hash_or_height(height.into()) .expect("should exist") .count() }; @@ -238,13 +234,19 @@ async fn scanning_zecpages_from_populated_zebra_state() -> Result<()> { .expect("position should fit in u32"), }; - let block_metadata = BlockMetadata::from_parts( - height.previous()?.0.into(), - BlockHash(block.header.previous_block_hash.0), - previous_sapling_tree_size - .try_into() - .expect("should fit in u32"), - ); + let block_metadata = if height.is_min() { + None + } else { + Some(BlockMetadata::from_parts( + height.previous()?.0.into(), + BlockHash(block.header.previous_block_hash.0), + db.sapling_tree_by_hash_or_height(block.header.previous_block_hash.into()) + .expect("should exist") + .count() + .try_into() + .expect("should fit in u32"), + )) + }; let compact_block = block_to_compact(block, chain_metadata); @@ -253,7 +255,7 @@ async fn scanning_zecpages_from_populated_zebra_state() -> Result<()> { compact_block.clone(), &vks[..], &[], - Some(&block_metadata), + block_metadata.as_ref(), ) .expect("should scan block successfully"); From 48a1d3a0ac7ec24728f7793acaa4b753bb76276e Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Wed, 8 Nov 2023 11:08:34 -0300 Subject: [PATCH 34/36] fix expect messages --- zebra-scan/src/tests.rs | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/zebra-scan/src/tests.rs b/zebra-scan/src/tests.rs index de66abaa3f5..5c7c8df857e 100644 --- a/zebra-scan/src/tests.rs +++ b/zebra-scan/src/tests.rs @@ -190,7 +190,7 @@ async fn scanning_zecpages_from_populated_zebra_state() -> Result<()> { let ivk = fvk.vk.ivk(); let vks: Vec<(&AccountId, &SaplingIvk)> = vec![(&account, &ivk)]; - let network = zebra_chain::parameters::Network::default(); + let network = zebra_chain::parameters::Network::Mainnet; // Create a continuous chain of mainnet blocks from genesis let blocks: Vec> = zebra_test::vectors::CONTINUOUS_MAINNET_BLOCKS @@ -211,27 +211,28 @@ async fn scanning_zecpages_from_populated_zebra_state() -> Result<()> { let mut transactions_scanned = 0; let mut blocks_scanned = 0; while let Some(block) = db.block(height.into()) { - // TODO: fix this issue in zcash_client_backend + // zcash_client_backend doesn't support scanning the genesis block, but that's ok, because + // Sapling activates at height 419,200. So we'll never scan these blocks in production code. let sapling_tree_size = if height.is_min() { 1 } else { db.sapling_tree_by_hash_or_height(height.into()) - .expect("should exist") + .expect("each state block must have a sapling tree") .count() }; let orchard_tree_size = db .orchard_tree_by_hash_or_height(height.into()) - .expect("should exist") + .expect("each state block must have a orchard tree") .count(); let chain_metadata = ChainMetadata { sapling_commitment_tree_size: sapling_tree_size .try_into() - .expect("position should fit in u32"), + .expect("sapling position is limited to u32::MAX"), orchard_commitment_tree_size: orchard_tree_size .try_into() - .expect("position should fit in u32"), + .expect("orchard position is limited to u32::MAX"), }; let block_metadata = if height.is_min() { @@ -241,10 +242,10 @@ async fn scanning_zecpages_from_populated_zebra_state() -> Result<()> { height.previous()?.0.into(), BlockHash(block.header.previous_block_hash.0), db.sapling_tree_by_hash_or_height(block.header.previous_block_hash.into()) - .expect("should exist") + .expect("each state block must have a sapling tree") .count() .try_into() - .expect("should fit in u32"), + .expect("sapling position is limited to u32::MAX"), )) }; @@ -257,7 +258,7 @@ async fn scanning_zecpages_from_populated_zebra_state() -> Result<()> { &[], block_metadata.as_ref(), ) - .expect("should scan block successfully"); + .expect("scanning block for the ZECpages viewing key should work"); transactions_found += res.transactions().len(); transactions_scanned += compact_block.vtx.len(); From 7d6ae238c3f905ed57c4976f7fd22907959ae9be Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 9 Nov 2023 08:26:50 +1000 Subject: [PATCH 35/36] Discard changes to Cargo.lock --- Cargo.lock | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 52b19e9e457..d1e05d657d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3046,12 +3046,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" -[[package]] -name = "powerfmt" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" - [[package]] name = "ppv-lite86" version = "0.2.17" From 0fc1c1f586f061937ec20062107ea80d2021a189 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 9 Nov 2023 08:26:55 +1000 Subject: [PATCH 36/36] Discard changes to deny.toml --- deny.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/deny.toml b/deny.toml index 2c37fbfc401..47466dc8c3d 100644 --- a/deny.toml +++ b/deny.toml @@ -75,9 +75,6 @@ skip-tree = [ # wait for indexmap, toml_edit, serde_json, tower to upgrade { name = "hashbrown", version = "=0.12.3" }, - # wait for jsonrpc-http-server to upgrade - { name = "redox_syscall", version = "=0.2.16" }, - # ECC crates # wait for hdwallet to upgrade