From 02898040caa51cb77a6347ebadf28969964e4e6e Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 9 Jun 2023 16:49:31 +1000 Subject: [PATCH 1/4] Abstract over minimal/mainnet spec in kzg crate --- Cargo.lock | 18 +- crypto/kzg/Cargo.toml | 3 +- crypto/kzg/src/kzg_commitment.rs | 12 +- crypto/kzg/src/kzg_proof.rs | 10 +- crypto/kzg/src/lib.rs | 292 ++++++++++++++++++++++++------- 5 files changed, 266 insertions(+), 69 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b8486344a20..db1e4f93dc1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -969,7 +969,20 @@ dependencies = [ [[package]] name = "c-kzg" version = "0.1.0" -source = "git+https://github.com/ethereum/c-kzg-4844?rev=fd24cf8e1e2f09a96b4e62a595b4e49f046ce6cf#fd24cf8e1e2f09a96b4e62a595b4e49f046ce6cf" +source = "git+https://github.com/michaelsproul//c-kzg-4844?rev=e652ce854a5f1855174cc89ce588513529656588#e652ce854a5f1855174cc89ce588513529656588" +dependencies = [ + "bindgen 0.64.0", + "cc", + "glob", + "hex", + "libc", + "serde", +] + +[[package]] +name = "c-kzg" +version = "0.1.0" +source = "git+https://github.com/michaelsproul/c-kzg-4844?rev=e652ce854a5f1855174cc89ce588513529656588#e652ce854a5f1855174cc89ce588513529656588" dependencies = [ "bindgen 0.64.0", "cc", @@ -3930,7 +3943,8 @@ name = "kzg" version = "0.1.0" dependencies = [ "arbitrary", - "c-kzg", + "c-kzg 0.1.0 (git+https://github.com/michaelsproul//c-kzg-4844?rev=e652ce854a5f1855174cc89ce588513529656588)", + "c-kzg 0.1.0 (git+https://github.com/michaelsproul/c-kzg-4844?rev=e652ce854a5f1855174cc89ce588513529656588)", "derivative", "ethereum_hashing", "ethereum_serde_utils", diff --git a/crypto/kzg/Cargo.toml b/crypto/kzg/Cargo.toml index 0fed5b41965..f1975d5ff49 100644 --- a/crypto/kzg/Cargo.toml +++ b/crypto/kzg/Cargo.toml @@ -16,7 +16,8 @@ serde_derive = "1.0.116" ethereum_serde_utils = "0.5.0" hex = "0.4.2" ethereum_hashing = "1.0.0-beta.2" -c-kzg = {git = "https://github.com/ethereum/c-kzg-4844", rev = "fd24cf8e1e2f09a96b4e62a595b4e49f046ce6cf" } +c-kzg = { git = "https://github.com/michaelsproul/c-kzg-4844", rev = "e652ce854a5f1855174cc89ce588513529656588" } +c_kzg_min = { package = "c-kzg", git = "https://github.com/michaelsproul//c-kzg-4844", rev = "e652ce854a5f1855174cc89ce588513529656588" } arbitrary = { version = "1.0", features = ["derive"], optional = true } [features] diff --git a/crypto/kzg/src/kzg_commitment.rs b/crypto/kzg/src/kzg_commitment.rs index 561bb10cdb8..20e7e80876b 100644 --- a/crypto/kzg/src/kzg_commitment.rs +++ b/crypto/kzg/src/kzg_commitment.rs @@ -1,4 +1,4 @@ -use c_kzg::{Bytes48, BYTES_PER_COMMITMENT}; +use c_kzg::BYTES_PER_COMMITMENT; use derivative::Derivative; use ethereum_hashing::hash_fixed; use serde::de::{Deserialize, Deserializer}; @@ -14,7 +14,7 @@ pub const BLOB_COMMITMENT_VERSION_KZG: u8 = 0x01; #[derive(Derivative, Clone, Copy, Encode, Decode)] #[derivative(PartialEq, Eq, Hash)] #[ssz(struct_behaviour = "transparent")] -pub struct KzgCommitment(pub [u8; BYTES_PER_COMMITMENT]); +pub struct KzgCommitment(pub [u8; c_kzg::BYTES_PER_COMMITMENT]); impl KzgCommitment { pub fn calculate_versioned_hash(&self) -> Hash256 { @@ -24,7 +24,13 @@ impl KzgCommitment { } } -impl From for Bytes48 { +impl From for c_kzg::Bytes48 { + fn from(value: KzgCommitment) -> Self { + value.0.into() + } +} + +impl From for c_kzg_min::Bytes48 { fn from(value: KzgCommitment) -> Self { value.0.into() } diff --git a/crypto/kzg/src/kzg_proof.rs b/crypto/kzg/src/kzg_proof.rs index 76035a4a870..ae1621c8c4b 100644 --- a/crypto/kzg/src/kzg_proof.rs +++ b/crypto/kzg/src/kzg_proof.rs @@ -1,4 +1,4 @@ -use c_kzg::{Bytes48, BYTES_PER_PROOF}; +use c_kzg::BYTES_PER_PROOF; use serde::de::{Deserialize, Deserializer}; use serde::ser::{Serialize, Serializer}; use ssz_derive::{Decode, Encode}; @@ -11,7 +11,13 @@ use tree_hash::{PackedEncoding, TreeHash}; #[ssz(struct_behaviour = "transparent")] pub struct KzgProof(pub [u8; BYTES_PER_PROOF]); -impl From for Bytes48 { +impl From for c_kzg::Bytes48 { + fn from(value: KzgProof) -> Self { + value.0.into() + } +} + +impl From for c_kzg_min::Bytes48 { fn from(value: KzgProof) -> Self { value.0.into() } diff --git a/crypto/kzg/src/lib.rs b/crypto/kzg/src/lib.rs index d0ed6728df5..e1b84b86fd0 100644 --- a/crypto/kzg/src/lib.rs +++ b/crypto/kzg/src/lib.rs @@ -2,30 +2,220 @@ mod kzg_commitment; mod kzg_proof; mod trusted_setup; +use std::ops::Deref; + pub use crate::{kzg_commitment::KzgCommitment, kzg_proof::KzgProof, trusted_setup::TrustedSetup}; -pub use c_kzg::{ - Blob, Error as CKzgError, KzgSettings, BYTES_PER_BLOB, BYTES_PER_FIELD_ELEMENT, - FIELD_ELEMENTS_PER_BLOB, -}; -use c_kzg::{Bytes32, Bytes48}; -use std::path::PathBuf; +pub use c_kzg::{Bytes32, Bytes48}; #[derive(Debug)] pub enum Error { - InvalidTrustedSetup(CKzgError), - InvalidKzgProof(CKzgError), - InvalidBytes(CKzgError), - KzgProofComputationFailed(CKzgError), - InvalidBlob(CKzgError), + InvalidTrustedSetup(CryptoError), + InvalidKzgProof(CryptoError), + InvalidBytes(CryptoError), + KzgProofComputationFailed(CryptoError), + InvalidBlob(CryptoError), +} + +#[derive(Debug)] +pub enum CryptoError { + CKzg(c_kzg::Error), + CKzgMin(c_kzg_min::Error), +} + +impl From for CryptoError { + fn from(e: c_kzg::Error) -> Self { + Self::CKzg(e) + } +} + +impl From for CryptoError { + fn from(e: c_kzg_min::Error) -> Self { + Self::CKzgMin(e) + } +} + +pub trait KzgPreset { + type KzgSettings; + type Blob; + type Bytes32: From<[u8; 32]> + Deref; + type Bytes48: From + From; + type Error: Into; + + const BYTES_PER_BLOB: usize; + const BYTES_PER_FIELD_ELEMENT: usize; + const FIELD_ELEMENTS_PER_BLOB: usize; + + fn bytes32_in(bytes: Bytes32) -> Self::Bytes32 { + let bytes: [u8; 32] = *bytes; + Self::Bytes32::from(bytes) + } + + fn bytes32_out(bytes: Self::Bytes32) -> Bytes32 { + let bytes: [u8; 32] = *bytes; + Bytes32::from(bytes) + } + + fn load_trusted_setup(trusted_setup: TrustedSetup) -> Result; + + fn compute_blob_kzg_proof( + blob: Self::Blob, + kzg_commitment: KzgCommitment, + trusted_setup: &Self::KzgSettings, + ) -> Result; + + fn verify_blob_kzg_proof( + blob: Self::Blob, + kzg_commitment: KzgCommitment, + kzg_proof: KzgProof, + trusted_setup: &Self::KzgSettings, + ) -> Result; + + fn verify_blob_kzg_proof_batch( + blobs: &[Self::Blob], + commitments_bytes: &[Self::Bytes48], + proofs_bytes: &[Self::Bytes48], + trusted_setup: &Self::KzgSettings, + ) -> Result; + + fn blob_to_kzg_commitment( + blob: Self::Blob, + trusted_setup: &Self::KzgSettings, + ) -> Result; + + fn compute_kzg_proof( + blob: Self::Blob, + z: Self::Bytes32, + trusted_setup: &Self::KzgSettings, + ) -> Result<(KzgProof, Self::Bytes32), CryptoError>; + + fn verify_kzg_proof( + kzg_commitment: KzgCommitment, + z: Self::Bytes32, + y: Self::Bytes32, + kzg_proof: KzgProof, + trusted_setup: &Self::KzgSettings, + ) -> Result; +} + +macro_rules! implement_kzg_preset { + ($preset_type:ident, $module_name:ident) => { + impl KzgPreset for $preset_type { + type KzgSettings = $module_name::KzgSettings; + type Blob = $module_name::Blob; + type Bytes32 = $module_name::Bytes32; + type Bytes48 = $module_name::Bytes48; + type Error = $module_name::Error; + + const BYTES_PER_BLOB: usize = $module_name::BYTES_PER_BLOB; + const BYTES_PER_FIELD_ELEMENT: usize = $module_name::BYTES_PER_FIELD_ELEMENT; + const FIELD_ELEMENTS_PER_BLOB: usize = $module_name::FIELD_ELEMENTS_PER_BLOB; + + fn load_trusted_setup( + trusted_setup: TrustedSetup, + ) -> Result { + $module_name::KzgSettings::load_trusted_setup( + trusted_setup.g1_points(), + trusted_setup.g2_points(), + ) + .map_err(CryptoError::from) + } + + fn compute_blob_kzg_proof( + blob: Self::Blob, + kzg_commitment: KzgCommitment, + trusted_setup: &Self::KzgSettings, + ) -> Result { + $module_name::KzgProof::compute_blob_kzg_proof( + blob, + kzg_commitment.into(), + trusted_setup, + ) + .map(|proof| KzgProof(proof.to_bytes().into_inner())) + .map_err(CryptoError::from) + } + + fn verify_blob_kzg_proof( + blob: Self::Blob, + kzg_commitment: KzgCommitment, + kzg_proof: KzgProof, + trusted_setup: &Self::KzgSettings, + ) -> Result { + $module_name::KzgProof::verify_blob_kzg_proof( + blob, + kzg_commitment.into(), + kzg_proof.into(), + trusted_setup, + ) + .map_err(CryptoError::from) + } + + fn verify_blob_kzg_proof_batch( + blobs: &[Self::Blob], + commitments_bytes: &[Self::Bytes48], + proofs_bytes: &[Self::Bytes48], + trusted_setup: &Self::KzgSettings, + ) -> Result { + $module_name::KzgProof::verify_blob_kzg_proof_batch( + blobs, + commitments_bytes, + proofs_bytes, + trusted_setup, + ) + .map_err(CryptoError::from) + } + + fn blob_to_kzg_commitment( + blob: Self::Blob, + trusted_setup: &Self::KzgSettings, + ) -> Result { + $module_name::KzgCommitment::blob_to_kzg_commitment(blob, trusted_setup) + .map(|com| KzgCommitment(com.to_bytes().into_inner())) + .map_err(CryptoError::from) + } + + fn compute_kzg_proof( + blob: Self::Blob, + z: Self::Bytes32, + trusted_setup: &Self::KzgSettings, + ) -> Result<(KzgProof, Self::Bytes32), CryptoError> { + $module_name::KzgProof::compute_kzg_proof(blob, z, trusted_setup) + .map(|(proof, y)| (KzgProof(proof.to_bytes().into_inner()), y)) + .map_err(CryptoError::from) + } + + fn verify_kzg_proof( + kzg_commitment: KzgCommitment, + z: Self::Bytes32, + y: Self::Bytes32, + kzg_proof: KzgProof, + trusted_setup: &Self::KzgSettings, + ) -> Result { + $module_name::KzgProof::verify_kzg_proof( + kzg_commitment.into(), + z, + y, + kzg_proof.into(), + trusted_setup, + ) + .map_err(CryptoError::from) + } + } + }; } +pub struct MainnetKzgPreset; +pub struct MinimalKzgPreset; + +implement_kzg_preset!(MainnetKzgPreset, c_kzg); +implement_kzg_preset!(MinimalKzgPreset, c_kzg_min); + /// A wrapper over a kzg library that holds the trusted setup parameters. #[derive(Debug)] -pub struct Kzg { - trusted_setup: KzgSettings, +pub struct Kzg { + trusted_setup: P::KzgSettings, } -impl Kzg { +impl Kzg

{ /// Load the kzg trusted setup parameters from a vec of G1 and G2 points. /// /// The number of G1 points should be equal to FIELD_ELEMENTS_PER_BLOB @@ -33,22 +223,7 @@ impl Kzg { /// The number of G2 points should be equal to 65. pub fn new_from_trusted_setup(trusted_setup: TrustedSetup) -> Result { Ok(Self { - trusted_setup: KzgSettings::load_trusted_setup( - trusted_setup.g1_points(), - trusted_setup.g2_points(), - ) - .map_err(Error::InvalidTrustedSetup)?, - }) - } - - /// Loads a trusted setup given the path to the file containing the trusted setup values. - /// The format is specified in `c_kzg::KzgSettings::load_trusted_setup_file`. - /// - /// Note: This function will likely be deprecated. Use `Kzg::new_from_trusted_setup` instead. - #[deprecated] - pub fn new_from_file(file_path: PathBuf) -> Result { - Ok(Self { - trusted_setup: KzgSettings::load_trusted_setup_file(file_path) + trusted_setup: P::load_trusted_setup(trusted_setup) .map_err(Error::InvalidTrustedSetup)?, }) } @@ -56,28 +231,22 @@ impl Kzg { /// Compute the kzg proof given a blob and its kzg commitment. pub fn compute_blob_kzg_proof( &self, - blob: Blob, + blob: P::Blob, kzg_commitment: KzgCommitment, ) -> Result { - c_kzg::KzgProof::compute_blob_kzg_proof(blob, kzg_commitment.into(), &self.trusted_setup) + P::compute_blob_kzg_proof(blob, kzg_commitment, &self.trusted_setup) .map_err(Error::KzgProofComputationFailed) - .map(|proof| KzgProof(proof.to_bytes().into_inner())) } /// Verify a kzg proof given the blob, kzg commitment and kzg proof. pub fn verify_blob_kzg_proof( &self, - blob: Blob, + blob: P::Blob, kzg_commitment: KzgCommitment, kzg_proof: KzgProof, ) -> Result { - c_kzg::KzgProof::verify_blob_kzg_proof( - blob, - kzg_commitment.into(), - kzg_proof.into(), - &self.trusted_setup, - ) - .map_err(Error::InvalidKzgProof) + P::verify_blob_kzg_proof(blob, kzg_commitment, kzg_proof, &self.trusted_setup) + .map_err(Error::InvalidKzgProof) } /// Verify a batch of blob commitment proof triplets. @@ -86,22 +255,21 @@ impl Kzg { /// TODO(pawan): test performance against a parallelized rayon impl. pub fn verify_blob_kzg_proof_batch( &self, - blobs: &[Blob], + blobs: &[P::Blob], kzg_commitments: &[KzgCommitment], kzg_proofs: &[KzgProof], ) -> Result { let commitments_bytes = kzg_commitments .iter() - .map(|comm| Bytes48::from_bytes(&comm.0)) - .collect::, _>>() - .map_err(Error::InvalidBytes)?; + .map(|comm| P::Bytes48::from(*comm)) + .collect::>(); let proofs_bytes = kzg_proofs .iter() - .map(|proof| Bytes48::from_bytes(&proof.0)) - .collect::, _>>() - .map_err(Error::InvalidBytes)?; - c_kzg::KzgProof::verify_blob_kzg_proof_batch( + .map(|proof| P::Bytes48::from(*proof)) + .collect::>(); + + P::verify_blob_kzg_proof_batch( blobs, &commitments_bytes, &proofs_bytes, @@ -111,17 +279,19 @@ impl Kzg { } /// Converts a blob to a kzg commitment. - pub fn blob_to_kzg_commitment(&self, blob: Blob) -> Result { - c_kzg::KzgCommitment::blob_to_kzg_commitment(blob, &self.trusted_setup) - .map_err(Error::InvalidBlob) - .map(|com| KzgCommitment(com.to_bytes().into_inner())) + pub fn blob_to_kzg_commitment(&self, blob: P::Blob) -> Result { + P::blob_to_kzg_commitment(blob, &self.trusted_setup).map_err(Error::InvalidBlob) } /// Computes the kzg proof for a given `blob` and an evaluation point `z` - pub fn compute_kzg_proof(&self, blob: Blob, z: Bytes32) -> Result<(KzgProof, Bytes32), Error> { - c_kzg::KzgProof::compute_kzg_proof(blob, z, &self.trusted_setup) + pub fn compute_kzg_proof( + &self, + blob: P::Blob, + z: Bytes32, + ) -> Result<(KzgProof, Bytes32), Error> { + P::compute_kzg_proof(blob, P::bytes32_in(z), &self.trusted_setup) .map_err(Error::KzgProofComputationFailed) - .map(|(proof, y)| (KzgProof(proof.to_bytes().into_inner()), y)) + .map(|(proof, y)| (proof, P::bytes32_out(y))) } /// Verifies a `kzg_proof` for a `kzg_commitment` that evaluating a polynomial at `z` results in `y` @@ -132,11 +302,11 @@ impl Kzg { y: Bytes32, kzg_proof: KzgProof, ) -> Result { - c_kzg::KzgProof::verify_kzg_proof( - kzg_commitment.into(), - z, - y, - kzg_proof.into(), + P::verify_kzg_proof( + kzg_commitment, + P::bytes32_in(z), + P::bytes32_in(y), + kzg_proof, &self.trusted_setup, ) .map_err(Error::InvalidKzgProof) From 59415cc66d7a6378af5378c11141130c95ed9804 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 9 Jun 2023 17:59:35 +1000 Subject: [PATCH 2/4] Start integrating new KZG --- .../test_utils/execution_block_generator.rs | 20 +++++++++------- .../src/test_utils/mock_execution_layer.rs | 2 +- .../execution_layer/src/test_utils/mod.rs | 4 ++-- consensus/types/src/eth_spec.rs | 13 ++++++++++ crypto/kzg/src/lib.rs | 24 ++++++++++++++++--- 5 files changed, 48 insertions(+), 15 deletions(-) diff --git a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs index 5e5508b6f8e..dc5184e95ae 100644 --- a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs +++ b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs @@ -8,7 +8,7 @@ use crate::{ }, BlobsBundleV1, ExecutionBlockWithTransactions, }; -use kzg::{Kzg, BYTES_PER_BLOB, BYTES_PER_FIELD_ELEMENT, FIELD_ELEMENTS_PER_BLOB}; +use kzg::{Kzg, KzgPreset}; use rand::RngCore; use serde::{Deserialize, Serialize}; use ssz::Encode; @@ -130,7 +130,7 @@ pub struct ExecutionBlockGenerator { * deneb stuff */ pub blobs_bundles: HashMap>, - pub kzg: Option>, + pub kzg: Option>>, } impl ExecutionBlockGenerator { @@ -140,7 +140,7 @@ impl ExecutionBlockGenerator { terminal_block_hash: ExecutionBlockHash, shanghai_time: Option, deneb_time: Option, - kzg: Option, + kzg: Option>, ) -> Self { let mut gen = Self { head_block: <_>::default(), @@ -635,29 +635,31 @@ impl ExecutionBlockGenerator { let mut transactions = vec![]; for blob_index in 0..n_blobs { // fill a vector with random bytes - let mut blob_bytes = [0u8; BYTES_PER_BLOB]; + let mut blob_bytes = vec![0u8; T::Kzg::BYTES_PER_BLOB]; rand::thread_rng().fill_bytes(&mut blob_bytes); // Ensure that the blob is canonical by ensuring that // each field element contained in the blob is < BLS_MODULUS - for i in 0..FIELD_ELEMENTS_PER_BLOB { - blob_bytes[i * BYTES_PER_FIELD_ELEMENT + BYTES_PER_FIELD_ELEMENT - 1] = 0; + for i in 0..T::Kzg::FIELD_ELEMENTS_PER_BLOB { + blob_bytes + [i * T::Kzg::BYTES_PER_FIELD_ELEMENT + T::Kzg::BYTES_PER_FIELD_ELEMENT - 1] = 0; } - let blob = Blob::::new(Vec::from(blob_bytes)) + let blob = Blob::::new(blob_bytes) .map_err(|e| format!("error constructing random blob: {:?}", e))?; + let kzg_blob = T::blob_from_bytes(&blob_bytes).unwrap(); let commitment = self .kzg .as_ref() .ok_or("kzg not initialized")? - .blob_to_kzg_commitment(blob_bytes.into()) + .blob_to_kzg_commitment(kzg_blob.clone()) .map_err(|e| format!("error computing kzg commitment: {:?}", e))?; let proof = self .kzg .as_ref() .ok_or("kzg not initialized")? - .compute_blob_kzg_proof(blob_bytes.into(), commitment) + .compute_blob_kzg_proof(kzg_blob, commitment) .map_err(|e| format!("error computing kzg proof: {:?}", e))?; let versioned_hash = commitment.calculate_versioned_hash(); diff --git a/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs b/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs index 0c6f5ce666e..e907f1cb04f 100644 --- a/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs +++ b/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs @@ -48,7 +48,7 @@ impl MockExecutionLayer { jwt_key: Option, spec: ChainSpec, builder_url: Option, - kzg: Option, + kzg: Option>, ) -> Self { let handle = executor.handle().unwrap(); diff --git a/beacon_node/execution_layer/src/test_utils/mod.rs b/beacon_node/execution_layer/src/test_utils/mod.rs index 72cd0e81e4e..b100d852133 100644 --- a/beacon_node/execution_layer/src/test_utils/mod.rs +++ b/beacon_node/execution_layer/src/test_utils/mod.rs @@ -104,7 +104,7 @@ impl MockServer { pub fn new_with_config( handle: &runtime::Handle, config: MockExecutionConfig, - kzg: Option, + kzg: Option>, ) -> Self { let MockExecutionConfig { jwt_key, @@ -185,7 +185,7 @@ impl MockServer { terminal_block_hash: ExecutionBlockHash, shanghai_time: Option, deneb_time: Option, - kzg: Option, + kzg: Option>, ) -> Self { Self::new_with_config( handle, diff --git a/consensus/types/src/eth_spec.rs b/consensus/types/src/eth_spec.rs index e7c00d19573..d8fe82103d7 100644 --- a/consensus/types/src/eth_spec.rs +++ b/consensus/types/src/eth_spec.rs @@ -1,5 +1,6 @@ use crate::*; +use kzg::{BlobTrait, KzgPreset, MainnetKzgPreset, MinimalKzgPreset}; use safe_arith::SafeArith; use serde_derive::{Deserialize, Serialize}; use ssz_types::typenum::{ @@ -51,6 +52,8 @@ impl fmt::Display for EthSpecId { pub trait EthSpec: 'static + Default + Sync + Send + Clone + Debug + PartialEq + Eq + for<'a> arbitrary::Arbitrary<'a> { + type Kzg: KzgPreset; + /* * Constants */ @@ -255,6 +258,10 @@ pub trait EthSpec: fn max_blobs_per_block() -> usize { Self::MaxBlobsPerBlock::to_usize() } + + fn blob_from_bytes(bytes: &[u8]) -> Result<::Blob, kzg::Error> { + ::Blob::from_bytes(bytes) + } } /// Macro to inherit some type values from another EthSpec. @@ -270,6 +277,8 @@ macro_rules! params_from_eth_spec { pub struct MainnetEthSpec; impl EthSpec for MainnetEthSpec { + type Kzg = MainnetKzgPreset; + type JustificationBitsLength = U4; type SubnetBitfieldLength = U64; type MaxValidatorsPerCommittee = U2048; @@ -318,6 +327,8 @@ impl EthSpec for MainnetEthSpec { pub struct MinimalEthSpec; impl EthSpec for MinimalEthSpec { + type Kzg = MinimalKzgPreset; + type SlotsPerEpoch = U8; type EpochsPerEth1VotingPeriod = U4; type SlotsPerHistoricalRoot = U64; @@ -369,6 +380,8 @@ impl EthSpec for MinimalEthSpec { pub struct GnosisEthSpec; impl EthSpec for GnosisEthSpec { + type Kzg = MainnetKzgPreset; + type JustificationBitsLength = U4; type SubnetBitfieldLength = U64; type MaxValidatorsPerCommittee = U2048; diff --git a/crypto/kzg/src/lib.rs b/crypto/kzg/src/lib.rs index e1b84b86fd0..4404e076bed 100644 --- a/crypto/kzg/src/lib.rs +++ b/crypto/kzg/src/lib.rs @@ -2,6 +2,8 @@ mod kzg_commitment; mod kzg_proof; mod trusted_setup; +use serde_derive::{Deserialize, Serialize}; +use std::fmt::Debug; use std::ops::Deref; pub use crate::{kzg_commitment::KzgCommitment, kzg_proof::KzgProof, trusted_setup::TrustedSetup}; @@ -34,9 +36,15 @@ impl From for CryptoError { } } -pub trait KzgPreset { - type KzgSettings; - type Blob; +pub trait BlobTrait: Sized + Clone { + fn from_bytes(bytes: &[u8]) -> Result; +} + +pub trait KzgPreset: + 'static + Default + Sync + Send + Clone + Debug + PartialEq + Eq + for<'a> arbitrary::Arbitrary<'a> +{ + type KzgSettings: Debug; + type Blob: BlobTrait; type Bytes32: From<[u8; 32]> + Deref; type Bytes48: From + From; type Error: Into; @@ -200,10 +208,20 @@ macro_rules! implement_kzg_preset { .map_err(CryptoError::from) } } + + impl BlobTrait for $module_name::Blob { + fn from_bytes(bytes: &[u8]) -> Result { + Self::from_bytes(bytes) + .map_err(CryptoError::from) + .map_err(Error::InvalidBlob) + } + } }; } +#[derive(Clone, PartialEq, Eq, Debug, Default, Serialize, Deserialize, arbitrary::Arbitrary)] pub struct MainnetKzgPreset; +#[derive(Clone, PartialEq, Eq, Debug, Default, Serialize, Deserialize, arbitrary::Arbitrary)] pub struct MinimalKzgPreset; implement_kzg_preset!(MainnetKzgPreset, c_kzg); From ae5d355ebbe1e957f17be3f74078815a34a2e334 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Tue, 27 Jun 2023 14:24:21 -0400 Subject: [PATCH 3/4] get ckzg working and fix some tests --- Cargo.lock | 8 +- beacon_node/beacon_chain/src/beacon_chain.rs | 38 +--- .../beacon_chain/src/blob_verification.rs | 19 +- .../src/data_availability_checker.rs | 4 +- beacon_node/beacon_chain/src/kzg_utils.rs | 35 ++-- beacon_node/beacon_chain/src/test_utils.rs | 53 ++++- .../tests/attestation_production.rs | 11 +- .../beacon_chain/tests/block_verification.rs | 194 ++++++++++++++---- beacon_node/beacon_chain/tests/store_tests.rs | 9 +- .../test_utils/execution_block_generator.rs | 55 +++-- .../minimal_testing_trusted_setups.json | 1 + common/eth2_network_config/src/lib.rs | 32 ++- crypto/kzg/Cargo.toml | 4 +- crypto/kzg/src/lib.rs | 32 ++- lcli/src/new_testnet.rs | 7 +- .../src/cases/kzg_blob_to_kzg_commitment.rs | 2 +- .../src/cases/kzg_compute_blob_kzg_proof.rs | 2 +- .../src/cases/kzg_compute_kzg_proof.rs | 2 +- .../src/cases/kzg_verify_blob_kzg_proof.rs | 10 +- .../cases/kzg_verify_blob_kzg_proof_batch.rs | 2 +- .../src/cases/kzg_verify_kzg_proof.rs | 2 +- 21 files changed, 368 insertions(+), 154 deletions(-) create mode 100644 common/eth2_network_config/built_in_network_configs/minimal_testing_trusted_setups.json diff --git a/Cargo.lock b/Cargo.lock index 3f80c2ff60d..283618becaf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -969,7 +969,7 @@ dependencies = [ [[package]] name = "c-kzg" version = "0.1.0" -source = "git+https://github.com/michaelsproul//c-kzg-4844?rev=e652ce854a5f1855174cc89ce588513529656588#e652ce854a5f1855174cc89ce588513529656588" +source = "git+https://github.com/realbigsean//c-kzg-4844?rev=1f25100d889c1dcc221f0c624225b38d511387a2#1f25100d889c1dcc221f0c624225b38d511387a2" dependencies = [ "bindgen 0.64.0", "cc", @@ -982,7 +982,7 @@ dependencies = [ [[package]] name = "c-kzg" version = "0.1.0" -source = "git+https://github.com/michaelsproul/c-kzg-4844?rev=e652ce854a5f1855174cc89ce588513529656588#e652ce854a5f1855174cc89ce588513529656588" +source = "git+https://github.com/realbigsean/c-kzg-4844?rev=1f25100d889c1dcc221f0c624225b38d511387a2#1f25100d889c1dcc221f0c624225b38d511387a2" dependencies = [ "bindgen 0.64.0", "cc", @@ -3952,8 +3952,8 @@ name = "kzg" version = "0.1.0" dependencies = [ "arbitrary", - "c-kzg 0.1.0 (git+https://github.com/michaelsproul//c-kzg-4844?rev=e652ce854a5f1855174cc89ce588513529656588)", - "c-kzg 0.1.0 (git+https://github.com/michaelsproul/c-kzg-4844?rev=e652ce854a5f1855174cc89ce588513529656588)", + "c-kzg 0.1.0 (git+https://github.com/realbigsean//c-kzg-4844?rev=1f25100d889c1dcc221f0c624225b38d511387a2)", + "c-kzg 0.1.0 (git+https://github.com/realbigsean/c-kzg-4844?rev=1f25100d889c1dcc221f0c624225b38d511387a2)", "derivative", "ethereum_hashing", "ethereum_serde_utils", diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 4793a6784e9..2423739ff4a 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -115,9 +115,8 @@ use store::{ use task_executor::{ShutdownReason, TaskExecutor}; use tokio_stream::Stream; use tree_hash::TreeHash; -use types::beacon_block_body::KzgCommitments; use types::beacon_state::CloneConfig; -use types::blob_sidecar::{BlobSidecarList, Blobs}; +use types::blob_sidecar::BlobSidecarList; use types::consts::deneb::MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS; use types::*; @@ -467,7 +466,7 @@ pub struct BeaconChain { pub genesis_backfill_slot: Slot, pub proposal_blob_cache: BlobCache, pub data_availability_checker: Arc>, - pub kzg: Option>, + pub kzg: Option::Kzg>>>, } type BeaconBlockAndState = (BeaconBlock, BeaconState); @@ -4929,7 +4928,7 @@ impl BeaconChain { //FIXME(sean) // - add a new timer for processing here - if let Some(blobs) = blobs_opt { + if let (Some(blobs), Some(proofs)) = (blobs_opt, proofs_opt) { let kzg = self .kzg .as_ref() @@ -4950,14 +4949,10 @@ impl BeaconChain { ))); } - let kzg_proofs = if let Some(proofs) = proofs_opt { - Vec::from(proofs) - } else { - Self::compute_blob_kzg_proofs(kzg, &blobs, expected_kzg_commitments, slot)? - }; + let kzg_proofs = Vec::from(proofs); kzg_utils::validate_blobs::( - kzg, + kzg.as_ref(), expected_kzg_commitments, &blobs, &kzg_proofs, @@ -5008,29 +5003,6 @@ impl BeaconChain { Ok((block, state)) } - fn compute_blob_kzg_proofs( - kzg: &Arc, - blobs: &Blobs, - expected_kzg_commitments: &KzgCommitments, - slot: Slot, - ) -> Result, BlockProductionError> { - blobs - .iter() - .enumerate() - .map(|(blob_index, blob)| { - let kzg_commitment = expected_kzg_commitments.get(blob_index).ok_or( - BlockProductionError::MissingKzgCommitment(format!( - "Missing KZG commitment for slot {} blob index {}", - slot, blob_index - )), - )?; - - kzg_utils::compute_blob_kzg_proof::(kzg, blob, *kzg_commitment) - .map_err(BlockProductionError::KzgError) - }) - .collect::, BlockProductionError>>() - } - /// This method must be called whenever an execution engine indicates that a payload is /// invalid. /// diff --git a/beacon_node/beacon_chain/src/blob_verification.rs b/beacon_node/beacon_chain/src/blob_verification.rs index ec7c3de58c9..8ae25551ddd 100644 --- a/beacon_node/beacon_chain/src/blob_verification.rs +++ b/beacon_node/beacon_chain/src/blob_verification.rs @@ -20,9 +20,9 @@ use ssz_types::FixedVector; use std::borrow::Cow; use types::blob_sidecar::{BlobIdentifier, FixedBlobSidecarList}; use types::{ - BeaconBlockRef, BeaconState, BeaconStateError, BlobSidecar, ChainSpec, CloneConfig, Epoch, - EthSpec, FullPayload, Hash256, KzgCommitment, RelativeEpoch, SignedBeaconBlock, - SignedBeaconBlockHeader, SignedBlobSidecar, Slot, + BeaconBlockRef, BeaconState, BeaconStateError, BlobSidecar, BlobSidecarList, ChainSpec, + CloneConfig, Epoch, EthSpec, FullPayload, Hash256, KzgCommitment, RelativeEpoch, + SignedBeaconBlock, SignedBeaconBlockHeader, SignedBlobSidecar, Slot, }; #[derive(Debug)] @@ -449,7 +449,7 @@ impl KzgVerifiedBlob { /// Returns an error if the kzg verification check fails. pub fn verify_kzg_for_blob( blob: Arc>, - kzg: &Kzg, + kzg: &Kzg, ) -> Result, AvailabilityCheckError> { let _timer = crate::metrics::start_timer(&crate::metrics::KZG_VERIFICATION_SINGLE_TIMES); //TODO(sean) remove clone @@ -469,7 +469,7 @@ pub fn verify_kzg_for_blob( /// in a loop since this function kzg verifies a list of blobs more efficiently. pub fn verify_kzg_for_blob_list( blob_list: Vec>>, - kzg: &Kzg, + kzg: &Kzg, ) -> Result, AvailabilityCheckError> { let _timer = crate::metrics::start_timer(&crate::metrics::KZG_VERIFICATION_BATCH_TIMES); let (blobs, (commitments, proofs)): (Vec<_>, (Vec<_>, Vec<_>)) = blob_list @@ -614,6 +614,15 @@ pub enum BlockWrapper { } impl BlockWrapper { + pub fn new(block: Arc>, blobs: Option>) -> Self { + match blobs { + Some(blobs) => { + let blobs = FixedVector::from(blobs.into_iter().map(Some).collect::>()); + BlockWrapper::BlockAndBlobs(block, blobs) + } + None => BlockWrapper::Block(block), + } + } pub fn deconstruct(self) -> (Arc>, Option>) { match self { BlockWrapper::Block(block) => (block, None), diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index 906cc681240..1d6a40e6c80 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -79,7 +79,7 @@ impl From for AvailabilityCheckError { pub struct DataAvailabilityChecker { availability_cache: Arc>, slot_clock: T::SlotClock, - kzg: Option>, + kzg: Option::Kzg>>>, spec: ChainSpec, } @@ -108,7 +108,7 @@ impl Availability { impl DataAvailabilityChecker { pub fn new( slot_clock: T::SlotClock, - kzg: Option>, + kzg: Option::Kzg>>>, store: BeaconStore, spec: ChainSpec, ) -> Result { diff --git a/beacon_node/beacon_chain/src/kzg_utils.rs b/beacon_node/beacon_chain/src/kzg_utils.rs index 7091b06fb8a..fddc97804ca 100644 --- a/beacon_node/beacon_chain/src/kzg_utils.rs +++ b/beacon_node/beacon_chain/src/kzg_utils.rs @@ -1,18 +1,16 @@ -use kzg::{Error as KzgError, Kzg, BYTES_PER_BLOB}; +use kzg::{Error as KzgError, Kzg, KzgPreset}; use types::{Blob, EthSpec, Hash256, KzgCommitment, KzgProof}; /// Converts a blob ssz List object to an array to be used with the kzg /// crypto library. -fn ssz_blob_to_crypto_blob(blob: Blob) -> kzg::Blob { - let blob_vec: Vec = blob.into(); - let mut arr = [0; BYTES_PER_BLOB]; - arr.copy_from_slice(&blob_vec); - arr.into() +fn ssz_blob_to_crypto_blob(blob: Blob) -> <::Kzg as KzgPreset>::Blob { + //TODO(sean) remove unwrap + T::blob_from_bytes(blob.to_vec().as_slice()).unwrap() } /// Validate a single blob-commitment-proof triplet from a `BlobSidecar`. pub fn validate_blob( - kzg: &Kzg, + kzg: &Kzg, blob: Blob, kzg_commitment: KzgCommitment, kzg_proof: KzgProof, @@ -26,11 +24,24 @@ pub fn validate_blob( /// Validate a batch of blob-commitment-proof triplets from multiple `BlobSidecars`. pub fn validate_blobs( - kzg: &Kzg, + kzg: &Kzg, expected_kzg_commitments: &[KzgCommitment], blobs: &[Blob], kzg_proofs: &[KzgProof], ) -> Result { + // TODO(sean) batch verification fails with a single element, it's unclear to me why + if blobs.len() == 1 && kzg_proofs.len() == 1 && expected_kzg_commitments.len() == 1 { + if let (Some(blob), Some(kzg_proof), Some(kzg_commitment)) = ( + blobs.get(0), + kzg_proofs.get(0), + expected_kzg_commitments.get(0), + ) { + return validate_blob::(kzg, blob.clone(), *kzg_commitment, *kzg_proof); + } else { + return Ok(false); + } + } + let blobs = blobs .iter() .map(|blob| ssz_blob_to_crypto_blob::(blob.clone())) // Avoid this clone @@ -41,7 +52,7 @@ pub fn validate_blobs( /// Compute the kzg proof given an ssz blob and its kzg commitment. pub fn compute_blob_kzg_proof( - kzg: &Kzg, + kzg: &Kzg, blob: &Blob, kzg_commitment: KzgCommitment, ) -> Result { @@ -51,7 +62,7 @@ pub fn compute_blob_kzg_proof( /// Compute the kzg commitment for a given blob. pub fn blob_to_kzg_commitment( - kzg: &Kzg, + kzg: &Kzg, blob: Blob, ) -> Result { kzg.blob_to_kzg_commitment(ssz_blob_to_crypto_blob::(blob)) @@ -59,7 +70,7 @@ pub fn blob_to_kzg_commitment( /// Compute the kzg proof for a given blob and an evaluation point z. pub fn compute_kzg_proof( - kzg: &Kzg, + kzg: &Kzg, blob: Blob, z: Hash256, ) -> Result<(KzgProof, Hash256), KzgError> { @@ -70,7 +81,7 @@ pub fn compute_kzg_proof( /// Verify a `kzg_proof` for a `kzg_commitment` that evaluating a polynomial at `z` results in `y` pub fn verify_kzg_proof( - kzg: &Kzg, + kzg: &Kzg, kzg_commitment: KzgCommitment, kzg_proof: KzgProof, z: Hash256, diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index d395d2ce93a..bf7abdb1a61 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -32,8 +32,8 @@ use int_to_bytes::int_to_bytes32; use kzg::{Kzg, TrustedSetup}; use merkle_proof::MerkleTree; use operation_pool::ReceivedPreCapella; -use parking_lot::Mutex; use parking_lot::RwLockWriteGuard; +use parking_lot::{Mutex, RwLock}; use rand::rngs::StdRng; use rand::Rng; use rand::SeedableRng; @@ -512,7 +512,7 @@ where }); let trusted_setup: TrustedSetup = - serde_json::from_reader(eth2_network_config::TRUSTED_SETUP) + serde_json::from_reader(eth2_network_config::get_trusted_setup::()) .map_err(|e| format!("Unable to read trusted setup file: {}", e)) .expect("should have trusted setup"); let kzg = Kzg::new_from_trusted_setup(trusted_setup).expect("should create kzg"); @@ -550,7 +550,7 @@ where HARNESS_GENESIS_TIME + spec.seconds_per_slot * E::slots_per_epoch() * epoch.as_u64() }); let trusted_setup: TrustedSetup = - serde_json::from_reader(eth2_network_config::TRUSTED_SETUP) + serde_json::from_reader(eth2_network_config::get_trusted_setup::()) .map_err(|e| format!("Unable to read trusted setup file: {}", e)) .expect("should have trusted setup"); let kzg = Kzg::new_from_trusted_setup(trusted_setup).expect("should create kzg"); @@ -608,7 +608,7 @@ where .validator_keypairs .expect("cannot build without validator keypairs"); let trusted_setup: TrustedSetup = - serde_json::from_reader(eth2_network_config::TRUSTED_SETUP) + serde_json::from_reader(eth2_network_config::get_trusted_setup::()) .map_err(|e| format!("Unable to read trusted setup file: {}", e)) .unwrap(); @@ -664,6 +664,7 @@ where runtime: self.runtime, mock_execution_layer: self.mock_execution_layer, mock_builder: self.mock_builder.map(Arc::new), + blob_signature_cache: <_>::default(), rng: make_rng(), } } @@ -690,9 +691,29 @@ pub struct BeaconChainHarness { pub mock_execution_layer: Option>, pub mock_builder: Option>>, + /// Cache for blob signature because we don't need them for import, but we do need them + /// to test gossip validation. We always make them during block production but drop them + /// before storing them in the db. + pub blob_signature_cache: Arc>>, + pub rng: Mutex, } +#[derive(Clone, Debug, Hash, Eq, PartialEq)] +pub struct BlobSignatureKey { + block_root: Hash256, + blob_index: u64, +} + +impl BlobSignatureKey { + pub fn new(block_root: Hash256, blob_index: u64) -> Self { + Self { + block_root, + blob_index, + } + } +} + pub type CommitteeAttestations = Vec<(Attestation, SubnetId)>; pub type HarnessAttestations = Vec<(CommitteeAttestations, Option>)>; @@ -885,10 +906,23 @@ where .proposal_blob_cache .pop(&signed_block.canonical_root()) { - let signed_blobs = Vec::from(blobs) + let signed_blobs: SignedBlobSidecarList = Vec::from(blobs) .into_iter() .map(|blob| { - blob.sign( + // Need to overwrite the proposer index before signing, not sure of a + // better way to do this. + let new_blob = BlobSidecar { + block_root: blob.block_root, + index: blob.index, + slot: blob.slot, + block_parent_root: blob.block_parent_root, + proposer_index: proposer_index as u64, + blob: blob.blob.clone(), + kzg_commitment: blob.kzg_commitment, + kzg_proof: blob.kzg_proof, + }; + let new_blob = Arc::new(new_blob); + new_blob.sign( &self.validator_keypairs[proposer_index].sk, &state.fork(), state.genesis_validators_root(), @@ -897,6 +931,13 @@ where }) .collect::>() .into(); + let mut guard = self.blob_signature_cache.write(); + for blob in &signed_blobs { + guard.insert( + BlobSignatureKey::new(blob.message.block_root, blob.message.index), + blob.signature.clone(), + ); + } (signed_block, Some(signed_blobs)) } else { (signed_block, None) diff --git a/beacon_node/beacon_chain/tests/attestation_production.rs b/beacon_node/beacon_chain/tests/attestation_production.rs index 0af59dedfea..97122c00043 100644 --- a/beacon_node/beacon_chain/tests/attestation_production.rs +++ b/beacon_node/beacon_chain/tests/attestation_production.rs @@ -68,6 +68,7 @@ async fn produces_attestations() { .store .make_full_block(&block_root, blinded_block) .unwrap(); + let blobs = chain.get_blobs(&block_root).unwrap(); let epoch_boundary_slot = state .current_epoch() @@ -132,7 +133,8 @@ async fn produces_attestations() { assert_eq!(data.target.epoch, state.current_epoch(), "bad target epoch"); assert_eq!(data.target.root, target_root, "bad target root"); - let block_wrapper: BlockWrapper = Arc::new(block.clone()).into(); + let block_wrapper = + BlockWrapper::::new(Arc::new(block.clone()), blobs.clone()); let beacon_chain::blob_verification::MaybeAvailableBlock::Available(available_block) = chain .data_availability_checker .check_availability(block_wrapper) @@ -202,7 +204,12 @@ async fn early_attester_cache_old_request() { .get_block(&head.beacon_block_root) .unwrap(); - let block_wrapper: BlockWrapper = head.beacon_block.clone().into(); + let head_blobs = harness + .chain + .get_blobs(&head.beacon_block_root) + .expect("should get blobs"); + + let block_wrapper = BlockWrapper::::new(head.beacon_block.clone(), head_blobs); let beacon_chain::blob_verification::MaybeAvailableBlock::Available(available_block) = harness.chain .data_availability_checker .check_availability(block_wrapper) diff --git a/beacon_node/beacon_chain/tests/block_verification.rs b/beacon_node/beacon_chain/tests/block_verification.rs index ce0ee940e3c..4b93a988864 100644 --- a/beacon_node/beacon_chain/tests/block_verification.rs +++ b/beacon_node/beacon_chain/tests/block_verification.rs @@ -1,6 +1,7 @@ #![cfg(not(debug_assertions))] use beacon_chain::blob_verification::BlockWrapper; +use beacon_chain::test_utils::BlobSignatureKey; use beacon_chain::{ blob_verification::AsBlock, test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy, EphemeralHarnessType}, @@ -33,7 +34,7 @@ lazy_static! { static ref KEYPAIRS: Vec = types::test_utils::generate_deterministic_keypairs(VALIDATOR_COUNT); } -async fn get_chain_segment() -> Vec> { +async fn get_chain_segment() -> (Vec>, Vec>>) { let harness = get_harness(VALIDATOR_COUNT); harness @@ -45,6 +46,7 @@ async fn get_chain_segment() -> Vec> { .await; let mut segment = Vec::with_capacity(CHAIN_SEGMENT_LENGTH); + let mut segment_blobs = Vec::with_capacity(CHAIN_SEGMENT_LENGTH); for snapshot in harness .chain .chain_dump() @@ -63,8 +65,76 @@ async fn get_chain_segment() -> Vec> { beacon_block: Arc::new(full_block), beacon_state: snapshot.beacon_state, }); + segment_blobs.push( + harness + .chain + .get_blobs(&snapshot.beacon_block_root) + .unwrap(), + ) + } + (segment, segment_blobs) +} + +async fn get_chain_segment_with_signed_blobs() -> ( + Vec>, + Vec, ::MaxBlobsPerBlock>>>, +) { + let harness = get_harness(VALIDATOR_COUNT); + + harness + .extend_chain( + CHAIN_SEGMENT_LENGTH, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::AllValidators, + ) + .await; + + let mut segment = Vec::with_capacity(CHAIN_SEGMENT_LENGTH); + let mut segment_blobs = Vec::with_capacity(CHAIN_SEGMENT_LENGTH); + for snapshot in harness + .chain + .chain_dump() + .expect("should dump chain") + .into_iter() + .skip(1) + { + let full_block = harness + .chain + .get_block(&snapshot.beacon_block_root) + .await + .unwrap() + .unwrap(); + segment.push(BeaconSnapshot { + beacon_block_root: snapshot.beacon_block_root, + beacon_block: Arc::new(full_block), + beacon_state: snapshot.beacon_state, + }); + let signed_blobs = harness + .chain + .get_blobs(&snapshot.beacon_block_root) + .unwrap() + .map(|blobs| { + let blobs = blobs + .into_iter() + .map(|blob| { + let block_root = blob.block_root; + let blob_index = blob.index; + SignedBlobSidecar { + message: blob, + signature: harness + .blob_signature_cache + .read() + .get(&BlobSignatureKey::new(block_root, blob_index)) + .unwrap() + .clone(), + } + }) + .collect::>(); + VariableList::from(blobs) + }); + segment_blobs.push(signed_blobs) } - segment + (segment, segment_blobs) } fn get_harness(validator_count: usize) -> BeaconChainHarness> { @@ -80,10 +150,14 @@ fn get_harness(validator_count: usize) -> BeaconChainHarness]) -> Vec>> { +fn chain_segment_blocks( + chain_segment: &[BeaconSnapshot], + blobs: &[Option>], +) -> Vec> { chain_segment .iter() - .map(|snapshot| snapshot.beacon_block.clone().into()) + .zip(blobs.into_iter()) + .map(|(snapshot, blobs)| BlockWrapper::new(snapshot.beacon_block.clone(), blobs.clone())) .collect() } @@ -139,8 +213,8 @@ fn update_parent_roots(snapshots: &mut [BeaconSnapshot]) { #[tokio::test] async fn chain_segment_full_segment() { let harness = get_harness(VALIDATOR_COUNT); - let chain_segment = get_chain_segment().await; - let blocks: Vec> = chain_segment_blocks(&chain_segment) + let (chain_segment, chain_segment_blobs) = get_chain_segment().await; + let blocks: Vec> = chain_segment_blocks(&chain_segment, &chain_segment_blobs) .into_iter() .map(|block| block.into()) .collect(); @@ -182,11 +256,12 @@ async fn chain_segment_full_segment() { async fn chain_segment_varying_chunk_size() { for chunk_size in &[1, 2, 3, 5, 31, 32, 33, 42] { let harness = get_harness(VALIDATOR_COUNT); - let chain_segment = get_chain_segment().await; - let blocks: Vec> = chain_segment_blocks(&chain_segment) - .into_iter() - .map(|block| block.into()) - .collect(); + let (chain_segment, chain_segment_blobs) = get_chain_segment().await; + let blocks: Vec> = + chain_segment_blocks(&chain_segment, &chain_segment_blobs) + .into_iter() + .map(|block| block.into()) + .collect(); harness .chain @@ -219,7 +294,7 @@ async fn chain_segment_varying_chunk_size() { #[tokio::test] async fn chain_segment_non_linear_parent_roots() { let harness = get_harness(VALIDATOR_COUNT); - let chain_segment = get_chain_segment().await; + let (chain_segment, chain_segment_blobs) = get_chain_segment().await; harness .chain @@ -229,10 +304,11 @@ async fn chain_segment_non_linear_parent_roots() { /* * Test with a block removed. */ - let mut blocks: Vec> = chain_segment_blocks(&chain_segment) - .into_iter() - .map(|block| block.into()) - .collect(); + let mut blocks: Vec> = + chain_segment_blocks(&chain_segment, &chain_segment_blobs) + .into_iter() + .map(|block| block.into()) + .collect(); blocks.remove(2); assert!( @@ -250,10 +326,11 @@ async fn chain_segment_non_linear_parent_roots() { /* * Test with a modified parent root. */ - let mut blocks: Vec> = chain_segment_blocks(&chain_segment) - .into_iter() - .map(|block| block.into()) - .collect(); + let mut blocks: Vec> = + chain_segment_blocks(&chain_segment, &chain_segment_blobs) + .into_iter() + .map(|block| block.into()) + .collect(); let (mut block, signature) = blocks[3].as_block().clone().deconstruct(); *block.parent_root_mut() = Hash256::zero(); @@ -275,7 +352,7 @@ async fn chain_segment_non_linear_parent_roots() { #[tokio::test] async fn chain_segment_non_linear_slots() { let harness = get_harness(VALIDATOR_COUNT); - let chain_segment = get_chain_segment().await; + let (chain_segment, chain_segment_blobs) = get_chain_segment().await; harness .chain .slot_clock @@ -285,10 +362,11 @@ async fn chain_segment_non_linear_slots() { * Test where a child is lower than the parent. */ - let mut blocks: Vec> = chain_segment_blocks(&chain_segment) - .into_iter() - .map(|block| block.into()) - .collect(); + let mut blocks: Vec> = + chain_segment_blocks(&chain_segment, &chain_segment_blobs) + .into_iter() + .map(|block| block.into()) + .collect(); let (mut block, signature) = blocks[3].as_block().clone().deconstruct(); *block.slot_mut() = Slot::new(0); blocks[3] = Arc::new(SignedBeaconBlock::from_block(block, signature)).into(); @@ -309,10 +387,11 @@ async fn chain_segment_non_linear_slots() { * Test where a child is equal to the parent. */ - let mut blocks: Vec> = chain_segment_blocks(&chain_segment) - .into_iter() - .map(|block| block.into()) - .collect(); + let mut blocks: Vec> = + chain_segment_blocks(&chain_segment, &chain_segment_blobs) + .into_iter() + .map(|block| block.into()) + .collect(); let (mut block, signature) = blocks[3].as_block().clone().deconstruct(); *block.slot_mut() = blocks[2].slot(); blocks[3] = Arc::new(SignedBeaconBlock::from_block(block, signature)).into(); @@ -413,7 +492,7 @@ async fn get_invalid_sigs_harness( } #[tokio::test] async fn invalid_signature_gossip_block() { - let chain_segment = get_chain_segment().await; + let (chain_segment, chain_segment_blobs) = get_chain_segment().await; for &block_index in BLOCK_INDICES { // Ensure the block will be rejected if imported on its own (without gossip checking). let harness = get_invalid_sigs_harness(&chain_segment).await; @@ -464,7 +543,7 @@ async fn invalid_signature_gossip_block() { #[tokio::test] async fn invalid_signature_block_proposal() { - let chain_segment = get_chain_segment().await; + let (chain_segment, chain_segment_blobs) = get_chain_segment().await; for &block_index in BLOCK_INDICES { let harness = get_invalid_sigs_harness(&chain_segment).await; let mut snapshots = chain_segment.clone(); @@ -498,7 +577,7 @@ async fn invalid_signature_block_proposal() { #[tokio::test] async fn invalid_signature_randao_reveal() { - let chain_segment = get_chain_segment().await; + let (chain_segment, chain_segment_blobs) = get_chain_segment().await; for &block_index in BLOCK_INDICES { let harness = get_invalid_sigs_harness(&chain_segment).await; let mut snapshots = chain_segment.clone(); @@ -518,7 +597,7 @@ async fn invalid_signature_randao_reveal() { #[tokio::test] async fn invalid_signature_proposer_slashing() { - let chain_segment = get_chain_segment().await; + let (chain_segment, chain_segment_blobs) = get_chain_segment().await; for &block_index in BLOCK_INDICES { let harness = get_invalid_sigs_harness(&chain_segment).await; let mut snapshots = chain_segment.clone(); @@ -559,7 +638,7 @@ async fn invalid_signature_proposer_slashing() { #[tokio::test] async fn invalid_signature_attester_slashing() { - let chain_segment = get_chain_segment().await; + let (chain_segment, chain_segment_blobs) = get_chain_segment().await; for &block_index in BLOCK_INDICES { let harness = get_invalid_sigs_harness(&chain_segment).await; let mut snapshots = chain_segment.clone(); @@ -611,7 +690,7 @@ async fn invalid_signature_attester_slashing() { #[tokio::test] async fn invalid_signature_attestation() { - let chain_segment = get_chain_segment().await; + let (chain_segment, chain_segment_blobs) = get_chain_segment().await; let mut checked_attestation = false; for &block_index in BLOCK_INDICES { @@ -648,7 +727,7 @@ async fn invalid_signature_attestation() { #[tokio::test] async fn invalid_signature_deposit() { - let chain_segment = get_chain_segment().await; + let (chain_segment, chain_segment_blobs) = get_chain_segment().await; for &block_index in BLOCK_INDICES { // Note: an invalid deposit signature is permitted! let harness = get_invalid_sigs_harness(&chain_segment).await; @@ -696,7 +775,7 @@ async fn invalid_signature_deposit() { #[tokio::test] async fn invalid_signature_exit() { - let chain_segment = get_chain_segment().await; + let (chain_segment, chain_segment_blobs) = get_chain_segment().await; for &block_index in BLOCK_INDICES { let harness = get_invalid_sigs_harness(&chain_segment).await; let mut snapshots = chain_segment.clone(); @@ -742,7 +821,7 @@ fn unwrap_err(result: Result) -> E { #[tokio::test] async fn block_gossip_verification() { let harness = get_harness(VALIDATOR_COUNT); - let chain_segment = get_chain_segment().await; + let (chain_segment, chain_segment_blobs) = get_chain_segment_with_signed_blobs().await; let block_index = CHAIN_SEGMENT_LENGTH - 2; @@ -752,7 +831,10 @@ async fn block_gossip_verification() { .set_slot(chain_segment[block_index].beacon_block.slot().as_u64()); // Import the ancestors prior to the block we're testing. - for snapshot in &chain_segment[0..block_index] { + for (snapshot, blobs_opt) in chain_segment[0..block_index] + .iter() + .zip(chain_segment_blobs.iter()) + { let gossip_verified = harness .chain .verify_block_for_gossip(snapshot.beacon_block.clone().into()) @@ -769,6 +851,21 @@ async fn block_gossip_verification() { ) .await .expect("should import valid gossip verified block"); + if let Some(blobs) = blobs_opt { + for blob in blobs { + let blob_index = blob.message.index; + let gossip_verified = harness + .chain + .verify_blob_sidecar_for_gossip(blob.clone(), blob_index) + .expect("should obtain gossip verified blob"); + + harness + .chain + .process_blob(gossip_verified, CountUnrealized::True) + .await + .expect("should import valid gossip verified blob"); + } + } } // Recompute the head to ensure we cache the latest view of fork choice. @@ -1026,14 +1123,29 @@ async fn verify_block_for_gossip_slashing_detection() { harness.advance_slot(); let state = harness.get_current_state(); - let ((block1, _), _) = harness.make_block(state.clone(), Slot::new(1)).await; - let ((block2, _), _) = harness.make_block(state, Slot::new(1)).await; + let ((block1, blobs1), _) = harness.make_block(state.clone(), Slot::new(1)).await; + let ((block2, _blobs2), _) = harness.make_block(state, Slot::new(1)).await; let verified_block = harness .chain .verify_block_for_gossip(Arc::new(block1).into()) .await .unwrap(); + + if let Some(blobs) = blobs1 { + for blob in blobs { + let blob_index = blob.message.index; + let verified_blob = harness + .chain + .verify_blob_sidecar_for_gossip(blob, blob_index) + .unwrap(); + harness + .chain + .process_blob(verified_blob, CountUnrealized::True) + .await + .unwrap(); + } + } harness .chain .process_block( diff --git a/beacon_node/beacon_chain/tests/store_tests.rs b/beacon_node/beacon_chain/tests/store_tests.rs index ea3ede4e191..81378e76752 100644 --- a/beacon_node/beacon_chain/tests/store_tests.rs +++ b/beacon_node/beacon_chain/tests/store_tests.rs @@ -13,7 +13,7 @@ use beacon_chain::{ migrate::MigratorConfig, BeaconChain, BeaconChainError, BeaconChainTypes, BeaconSnapshot, ChainConfig, NotifyExecutionLayer, ServerSentEventHandler, WhenSlotSkipped, }; -use eth2_network_config::TRUSTED_SETUP; +use eth2_network_config::get_trusted_setup; use execution_layer::auth::JwtKey; use execution_layer::test_utils::{MockExecutionLayer, DEFAULT_JWT_SECRET, DEFAULT_TERMINAL_BLOCK}; use fork_choice::CountUnrealized; @@ -2117,9 +2117,10 @@ async fn weak_subjectivity_sync() { let store = get_store(&temp2); let spec = test_spec::(); let seconds_per_slot = spec.seconds_per_slot; - let trusted_setup: TrustedSetup = serde_json::from_reader(TRUSTED_SETUP) - .map_err(|e| println!("Unable to read trusted setup file: {}", e)) - .unwrap(); + let trusted_setup: TrustedSetup = + serde_json::from_reader(get_trusted_setup::<::Kzg>()) + .map_err(|e| println!("Unable to read trusted setup file: {}", e)) + .unwrap(); let spec = harness.spec.clone(); let shanghai_time = spec.capella_fork_epoch.map(|epoch| { diff --git a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs index 0fca62792c5..a281b7d2be1 100644 --- a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs +++ b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs @@ -8,6 +8,7 @@ use crate::{ }, ethers_tx_to_bytes, BlobsBundleV1, ExecutionBlockWithTransactions, }; +use ethers_core::types::Transaction; use kzg::{Kzg, KzgPreset}; use rand::RngCore; use serde::{Deserialize, Serialize}; @@ -404,8 +405,7 @@ impl ExecutionBlockGenerator { } pub fn get_blobs_bundle(&mut self, id: &PayloadId) -> Option> { - // remove it to free memory - self.blobs_bundles.remove(id) + self.blobs_bundles.get(id).cloned() } pub fn new_payload(&mut self, payload: ExecutionPayload) -> PayloadStatusV1 { @@ -625,15 +625,15 @@ impl ExecutionBlockGenerator { } } -pub fn generate_random_blobs( +pub fn generate_random_blobs( n_blobs: usize, - kzg: &Kzg

, + kzg: &Kzg, ) -> Result<(BlobsBundleV1, Transactions), String> { let mut bundle = BlobsBundleV1::::default(); let mut transactions = vec![]; for blob_index in 0..n_blobs { // fill a vector with random bytes - let mut blob_bytes =vec![0u8; T::Kzg::BYTES_PER_BLOB]; + let mut blob_bytes = vec![0u8; T::Kzg::BYTES_PER_BLOB]; rand::thread_rng().fill_bytes(&mut blob_bytes); // Ensure that the blob is canonical by ensuring that // each field element contained in the blob is < BLS_MODULUS @@ -641,26 +641,39 @@ pub fn generate_random_blobs( blob_bytes[i * T::Kzg::BYTES_PER_FIELD_ELEMENT] = 0; } - let blob = Blob::::new(Vec::from(blob_bytes)) - .map_err(|e| format!("error constructing random blob: {:?}", e))?; - let kzg_blob = T::blob_from_bytes(&blob_bytes).unwrap(); - + let blob = Blob::::new(Vec::from(blob_bytes)) + .map_err(|e| format!("error constructing random blob: {:?}", e))?; + let kzg_blob = T::blob_from_bytes(&blob).unwrap(); let commitment = kzg - .as_ref() - .ok_or("kzg not initialized")? - .blob_to_kzg_commitment(kzg_blob.clone()) - .map_err(|e| format!("error computing kzg commitment: {:?}", e))?; + .blob_to_kzg_commitment(kzg_blob.clone()) + .map_err(|e| format!("error computing kzg commitment: {:?}", e))?; - let proof = kzg - .as_ref() - .ok_or("kzg not initialized")? - .compute_blob_kzg_proof(kzg_blob, commitment) - .map_err(|e| format!("error computing kzg proof: {:?}", e))?; + let proof = kzg + .compute_blob_kzg_proof(kzg_blob, commitment) + .map_err(|e| format!("error computing kzg proof: {:?}", e))?; // Calculate transaction bytes. We don't care about the contents of the transaction. - let tx_bytes = ethers_core::types::Transaction::default(); - let tx = ethers_tx_to_bytes::(tx_bytes) + let tx: Transaction = serde_json::from_str( + r#"{ + "blockHash":"0x1d59ff54b1eb26b013ce3cb5fc9dab3705b415a67127a003c3e61eb445bb8df2", + "blockNumber":"0x5daf3b", + "from":"0xa7d9ddbe1f17865597fbd27ec712455208b6b76d", + "gas":"0xc350", + "gasPrice":"0x4a817c800", + "hash":"0x88df016429689c079f3b2f6ad39fa052532c56795b733da78a91ebe6a713944b", + "input":"0x68656c6c6f21", + "nonce":"0x15", + "to":"0xf02c1c8e6114b1dbe8937a39260b5b0a374432bb", + "transactionIndex":"0x41", + "value":"0xf3dbb76162000", + "v":"0x25", + "r":"0x1b5e176d927f8e9ab405058b2d2457392da3e20f328b16ddabcebc33eaac5fea", + "s":"0x4ba69724e8f69de52f0125ad8b3c5c2cef33019bac3249e2c0a2192766d1721c" + }"#, + ) + .unwrap(); + let tx = ethers_tx_to_bytes::(tx) .map_err(|e| format!("error converting tx bytes to SSZ: {:?}", e))?; transactions.push(tx); @@ -690,7 +703,7 @@ pub fn generate_pow_block( terminal_block_number: u64, block_number: u64, parent_hash: ExecutionBlockHash, -) -> Result { +) -> Result { if block_number > terminal_block_number { return Err(format!( "{} is beyond terminal pow block {}", diff --git a/common/eth2_network_config/built_in_network_configs/minimal_testing_trusted_setups.json b/common/eth2_network_config/built_in_network_configs/minimal_testing_trusted_setups.json new file mode 100644 index 00000000000..4eda1d314d2 --- /dev/null +++ b/common/eth2_network_config/built_in_network_configs/minimal_testing_trusted_setups.json @@ -0,0 +1 @@ +{"setup_G1": ["0x97f1d3a73197d7942695638c4fa9ac0fc3688c4f9774b905a14e3a3f171bac586c55e83ff97a1aeffb3af00adb22c6bb", "0x854262641262cb9e056a8512808ea6864d903dbcad713fd6da8dddfa5ce40d85612c912063ace060ed8c4bf005bab839", "0x86f708eee5ae0cf40be36993e760d9cb3b2371f22db3209947c5d21ea68e55186b30871c50bf11ef29e5248bf42d5678", "0x94f9c0bafb23cbbf34a93a64243e3e0f934b57593651f3464de7dc174468123d9698f1b9dfa22bb5b6eb96eae002f29f"], "setup_G2": ["0x93e02b6052719f607dacd3a088274f65596bd0d09920b61ab5da61bbdc7f5049334cf11213945d57e5ac7d055d042b7e024aa2b2f08f0a91260805272dc51051c6e47ad4fa403b02b4510b647ae3d1770bac0326a805bbefd48056c8c121bdb8", "0x99aca9fb2f7760cecb892bf7262c176b334824f5727f680bba701a33e322cb6667531410dfc7c8e4321a3f0ea8af48cb1436638a2093123f046f0f504cc2a864825542873edbbc5d7ed17af125a4f2cf6433c6f4f61b81173726981dd989761d", "0x88e2e982982bf8231e747e9dfcd14c05bd02623d1332734d2af26246c6869fb56ee6c994843f593178a040495ba61f4a083b0e18110b1d9f5224783d8f9a895e8ee744e87929430e9ba96bd29251cbf61240b256d1525600f3d562894d93d659", "0xa2d33775e3d9e6af0d1b27d389e6c021a578e617a3d6627686db6288d4b3dffd7a847a00f7ef01828b7f42885b660e4204923402aca18fbae74ccd4e9c50dd8c2281b38dc09c022342ed1ac695d53f7081cb21f05fdfc0a3508c04759196fcd3", "0xaf565445d2ad54c83a75c40e8895f5ad7219a8c728bce9d58d7a83716e095432993ebbd3f6911c66415a6f920d1a4d171478509b54a114308a020b33bf4487a7a8d0aa76ae4676a9b54e765a680f562d3a4fcb2e92c58b14b49b5b2917cc258f", "0x8aa99cfaf514cef4801599cadd780d222194ca1ad69a34779c2bcfda93e5dbeb931e13914421b5809a6c81f12cf7038b04a35257cc9e94c33761e68565b1274aa6a6f9d66477229747a66b308b138f92aa4326a3bf23df65a1fe33b3b289bfe1", "0x99ba36d8b4f56bde026099278548b1afc0a987cbd7c9baa51fc8e6cbb8237a17636f1a44a385cec69b05a5802059956a11fe793cabb939c38800f9c239ca2518e898ade1ec2513c9ee492071a35aabd78182392a09123d28dbc233313c9120c4", "0xa7dc40c36afccb30a2eaff250860b28b227c195cf05674704c567d77d6655c446ae835f8fc8667e71147ab02afcb2dad0babe60cbfa37d7c2cddc68d2dec54f28a4142f8353590a3902d5ddaa22066ab563dd1435dda83f276387b9767d69120", "0x939e6cc97a8b88572852a5b7f25e4838556307f60aeafb5d2b6961edbcafd4b48cb6ac980ffbacf4be963f324ba81e3d12de4f1459d8c746d0762c66ae1b166027f7fbe641d9c48f3c7d97b06d956b0be51dcc9aab65f3e99e1388e63bdd79f9", "0xb391e156541dfd4003d1697cdb7ec815b309807320574906b2e652ef0175828b356d215cd374b1b34d9f470b3fa0e643113e67b2273268f922f04f072cfb89008358185b25cd631f82911a3f20f90f75758ffb99bebb8076458ae1e9d1ae898c", "0xb9ac9c84934cc2a85c876eff65577e1dfce1935cd6392c877dd881a7d2f5c3e9344f28c04f90c62a6db4237ca00f9e0d00cb5f63e3f060fc7303916e19273b6fe455f331cabbe2fe5a22d584484f0d4176120fec9819fbb0a01e6d38695acfcd", "0x88209eb030c5d78734bf2c2a5c539653fd3c24b4c08e624f9ddc4a6550efbdc1054a56eb0c807595aad6de56fda326aa196d032a8b4b48d40140a2d77df3c7243eda6507936389a321a5811eb38e32ee433c788deeae1eb928b00940e2944bcc", "0xa8632ddc9cf7cbc1e8b74a05b7d4a89618c64afe30367ca0c9550ae7d320bf4e51c5a69e1501a1d8bee4240d13d7835501aa39fdc401a74f4d5734e268a7ce29a1fcfdb0a8bc64e0dd4a9e8578d6985bc2bc6a3764ce7a3703f6fb2e52557a2b", "0xa037ac67e8bb6f4193ac967e05d080a489f58ef8d3d30a89798246f3e4936121ee445b03e410a09e8ebc0db2e2477d110aad0ade99b0887f1eb016e750f42135866907f150bd6f4f99a8cb94281474166874808ebe03b118c5daab16dafdc38b", "0xa50d9143116bffa3b237da8e1805327e81e9cd25e658289bd727d5f9e0020172cc8690dcfe31a240e5cbc48353b88c4908baa1dd7320165556e0aa633f62fcbe7870222d345a3bbcdb7ab6c07f0fd86be559964afabf56f0a8cbc0b4b91d477e", "0xafa988ea6fa4f40c5ad07d2d580d29025ddf56d6ef1171a8b8de3464203f70b97d6f5ace72747345204b35150e06154d1477516a989ce8eea7871cc0d0de00a077c0fb23ad4837e409d0b885bf3f2dde11a30fa6273d662e68e09f461e52932f", "0x97fa1a943ed8b81574304a3d03f4f15907f6e6e0cd36a66bd2ad2c75afafc70a61d3ff69b77ebe4dae9ca0fcedef80081062705e60bbb6ea0f1f398c84d2f8e4a3ac142ac66426c21ad5e9994ebbcc406af474c4aec5e32fadcb21875af7c9f1", "0xb30a564614493886f14a5dd71c89457504f8c59a7ac01b665ed167e9a8f9ee5832198fd319ecd234196ee57031bdf3840bd5a923e203a1938bc795c704b5285389750e1fd10d7050061ba19db00a60a2c0384a7d661d7d48ebe6962272230859", "0x84c8dea942cfae71cb02e705ec496d967425793ce8812e7ee53c2f23713abeaff566a658cd1c73dfd18187d16253a6ee0a623e82cd18e31cd1a1875d19c078835dc9292e141686150a88065226ada264740143e87c03a0f6c4da8c187438ebf4", "0x8c3abae8aed60338f8c4ff80aab22f8a2ae56756a93566c906f490a97151d34a1c3318054e1c494c60cc53327ad86a2d02c6c76a406726ce4f88635bc32eff0db0b61762dc518b95fa8da82e87e4bf3de54f1d72180ef53ed7bc5413e6a9a510", "0xa328230c92a6b1cef6a444bcb64edb992f71e3d7b93f0b6b8b408ba7c908db746d92ddb2c7588bab438ef3bc61be1c2f0dfc86ba2ff514b42b35c80f89b2e780f813ea1dfb977fbded2cd9b553b747fa952e227ebd8f071163d421fc337f04c9", "0xb482cab423cd5f1c5df036070aade7aa016283d69619d664025c3feab866a0a5691d344b2ee2bedc5dedd1f9a73eae16003a3827c9e5bbe22ded32d848fba840ffad1141ad158f5c40bc8ae0d03781b9705d851a7f1391b096c576c0f4f2a6b0", "0x919ee1df27fabcb21237a1b7b98f53d41d849e1b6a8f9e28c3fae2841c6b5a250e4041c737e6725476e5cd715e34d3880f58d80f61efaabc261bdc703e8750f48a923e9bf8980931b9fd9e40014c66c54b3e7c98241d76d1aa47af43313a65a1", "0xac94830145dbe9a8f7e6e0fc1f5fb454502d22abcafdc2dd96c6933c604461fa83b2b37385f4bc454875a02a6d4157841250956783515d11c7456e7f11b745f12856d89f5feedaf6a61a483a6c33a21cd2ba0c18eb41a1a2e7fc33bb53e4c570", "0xb209c699f1233735c5bb4bce848e4365fd76651ae2184d2279a90df0c2f69ffa2a24d84a9b9f274021072953c0d65e1a0202d490d6c37186af240114e445d87bff754b4824937e4f2c90a574061b1c4910fed88d90f698025a2a264e656cb8a4", "0x93320dc0576b0d069de63c40e5582b4486d9adf5e69e77e3ebaf3da26976fe42147a65051501bc8383f99e7ba75479c70a6726c2cd08bf98c7481f1f819712292d833a879f21a1221a9610bc748fb5e911055122fdb4055cdc84e8bfe0f4df9b", "0xa4380b240e998cdf668591f71a0c88ed143b0185a920787627ce65095f8223dc606fa5bce93377af100de92d663e675c0736d7f1973603a84a5c4162fb5e01c88c7493503ae1d7e9fbe8ece9b418397d68c21eeb88dae226e09875d372c646dd", "0xaab48517d69135a16b36b685adfe9b2544a709135a21ba3e75981a2cba4ec81d1fe28ac0f72fde0c0001c15300ed6a810f58d3117bdd58d0149751d6508cf8a1a1ff7b63dd02d2730a9d6fe96c77c502fe8ed46d50a181ec4bb35e37dfbd6af4", "0x8277265fe75ab89ce4ec65b33fb4084bec0a56d81faf2f7a9070d2ca3065678e03a790350eba56323a54e0285bc32fe8007d5259740fde226e16cbde8354eacd562294eb9b7f727ed72ffbdad86f467cf057c737b34b80a41deb92634ed866f5", "0xaa40a24cb2ebe606d969392c03020070f044c95088d80f57f771b837c048342d2cd3474600d7660441090ffb8d2ffb7f0eddd67eb378e3e1477a6ba0bc38096d5d2d3355bc8b60f605f57f0c1899da591457440352381d2b38c0aa9acc7fe419", "0x80815d10685808cb630820629bcd2fa9041c9b74433630c0b9c1b7f7e8edf1440b520217f76ec9a50c125cf4438aa66006a1928a9ed2321da7ea325c3d56b65462b72118ca2c99a0ea733aa11da9abbeda6cc71ffeed301ae70213a29e697dcd", "0xac235d079f91b00b1fead7523da8f73a5409fa8970907af0c5d5e4c6a0996dccfcdb0d822d08c7fbc0c24799457d011d04312d20831825f23cf988141056a6814c8a1cac9efe37bdcbfa272aed24cd92810fea7c49b0d07683a5c53643872179", "0xb8aa59534d75fa5ac1c2c3f963bf73899aff5210059dbde8a8635561c6249e5143affee3bd2fd57575213b52d9a73d5702525867a7dcbb1d0a49b98c2925556fc5463ff0209742046a24ab29e74257d6419401093cc4371944d811cc300b6a67", "0x80bbfc5b816eea29a6d84e2217dee4d547306994d39e5592515e1b0807b67fe960d1d5addb0ff1a20c158bdb294c04bf093d28996121845a2c9268e2c9ac0f4067e889c6aaca62f8535d35b45036954bd069e3afa84f04721538c26003304c20", "0xa535c17d0e151d0e03d42dd58ba8c715bee3fabca2890e0e016071d34184b6b34e770d2be29c8ec76b69bcc471d50f4d043c2c240e9b93a81cff7ee2724e02018dfd9b534e40be641fdb4884abcd83b76f517557ffba508f1ba2f56313f4de94", "0xb237eb7465df0d325a3aa58269be2627e4978f9863f4f100ed4c303cb1f6549e606f2e3c9180824d8049191965c8dacd0a0c76cc56cb22cf1bcfdb39372c8aa29b4f7b34582b1719e6bd59c930d87d5ccd838743b585d6e229d5ed42337315c0", "0x805c335a2a9d2de30809cf30808ef836d88e9453c510716f01696f14c72dd60505eca8f128970edc8e63a9aa1f8792ac0dd50dcc84fbf4cc8b32349c682a6a27bc7551c7aa273a94c1606d07710188d93579afe3be1781bded15a34ed6047922", "0xb25dadf385ddd3c39bcb0a014d3d4f66127946b1aceae8809e3a03d66cc25e27142ca108316391f857fe82fdea4db2520cc73793b695eafbf3ade00ef7ec747b0457e49303f5e1a370f5263b436566fe24a0876e5fe088238c7be37a0718d65f", "0xb0f753081cabe2c8fce73aba82ff67dbc9842598b3e7fa3ce2a1f534536f8ac63c532fe66552ac6b7adb28c73ed4c8a4184849be7c1756a4681ce29ebf5e1c3aa806b667ee6bd68f6397aba3215dc1caec6742f21d681e32cd1160d6a3b1d7ee", "0xb798771eeb3d7a17c62ba5916cc034bba870da6b1ac14c2e1cae71af3ad4e0c0d1ff983f691e0e55289d5a33b131f2ec12430c9566dd71f4d8be9c79155357a5c30c5efcfd75bbe1bb6d5ada4d50604ea49ed838d3641f268ca6e25c9c4b6b72", "0xb52554c017388b099804abbe565346591a086d9979e10140ddaccc0a3680e506db775d7cbeafde67563adf0f09f5c2420caf19629f4e8f03e6fe02e9416ecd5269989e482b90004a083967d1141387eb74865bac6bd17e7a6d5f58225e52d4b7", "0xb520ff694520919023d44d53f98a7de2f78ff37b2d9193dcaa35556a6a0febf767781a4c961dce7c804bfdf81935f8f0082865253da52e79dfa1c5ff74d61495b2da76e167d46114709e877a7791a3a95e33a42f56b83f5f5afe271c67ae997c", "0xb721401983440797a03d5b99f2088a0b249aa911969c34dd6c615b0060325da555d2ad99d931170c0868b0488a2234a4114cc0013d5163b833f5c45c5eb536421c016cf85788390176bb2dc4c196d6be26bbbfceae048b82f0d8039222e71c94", "0xacd9d833ba0a8cbd8d1ba939a11ea0fa5607e1bc6e693ec318bdb097aedd042d76e695dcebebd142e2e4ac30b1905dff03ec36d9cc70577e4dbe5e9ed7c20c7afb13a7f0155f203c6b83b9f1ad3d20a0d4aef0fbbbcf466ffc1bcd482bc2f5e0", "0x8cc1795de015f2b0e72116f169f3b4624b7738ceebea354e0bd9051c27b86f647ea36cad57ea6884c1a8adf9b45cd83514fa687e68878bbd613d793aa10986d5a0411f081689229e0d72133b3667b9f3f1a02211d0e680564eb1ea43393e1f36", "0xaa9281c61113c343a108de1036570feefc72fb7a96ff11f73024de12b83f29631f5a8a5900e6f10b15227c6f7462881511271bf785ebdf95ce288100e5dab391f664f6ff76c72b65b34479a4f43e5e8eba292209d6654157286ad3242ac342db", "0xaaf16866275082e59d415db317aa874267d048ee405a553e852e6d175711d31a1fee99912345915bce121f43bc3e00d81338e5fcd3c8a1012fb4f172a9fe15622dd368b4d9d5cb60d189f423b071791fe26cea7676aca8df07965cacf80b0cd0", "0xaccc80b3d8a6ffa648487a3d3c0ce1aeeb5401edf3cf2e385ea4a6d5fc110054fcce38f01f1da7141bbed30eb7a0a6810c82212bbb9da75d6033082dbcf6bc6a5791f85aa0f045a10da5de015edbf369b4d23b32b0c058962d2ee88e6911f994", "0x83f1089395a16077738cc7c9a6d6a3dc9033aac4abc508af5a1f007ca92e1a80b2e6f2dbda7fdcf0d5646de790a6201d0a9cfbcb6620a1426600e3a6a425ec004384f49fb9dcd166691a47177d45dcbcb761a11d46220b0aa09fc946131f7aa5", "0x9246bb586d43cb817c2e15ed609156e9f1cd284ba2f4797bbfa51c0341e1ba382eaac059aa9f63fb88d228a1a932839a171e7c7d00199dc7c4d6c5ea038a02cbc3cc5297c70401520e70ebbcffacd6a703f62896f3c788f94dde3c33ab0ecbdb", "0xa316cb7c74feb0563c56cc79015e2774fbeca458bf8e9fb07894f9d6bcd73f7fb9428e87c816e5629e4bf7f3ec567fbc091549471b75492dde08217cb334b716b4582b24384586e53388873a78a90ec01bd7c3bace9cfc52161467df16e27c33", "0xade18c74bbe60d1d69f4a570f8e5fd8696c26cc9e02829040b6b14cb9c49a4b3263b5bd5e16ec0b29010b4be054c16ab09304e23442af7d7f5fcc60bc6c5634ab6e4aed7ef334b2785e4c7672d59a687278e42d310342db5e5975d716e6d1595", "0xb7728800bb2039acf228fa3d8028569c426cb85d28b2b5820bbef938d5ca8c4df981d3e01a309e26ca101e8295d0f6990c03b8c239798323575874a4ee5bfe46cfe99b9657189142aacd8f8d1f26cf4c0e73c6397c31ba8f18102b9ea315b638", "0x8fb14f2a9be193f54977ecd3021663108ea143627b9a9d9faff85d1a86b855f6c437eab435fad3304f245bd7732af07f1173494cdb802fb96e85d2db89e1643206e183f3b228ca8d3f586e71aa9308eaf0223100bf07942fc39e465016d1f775", "0xac1e025e53d98fdb3380489dce82d9d4bd3a6c98f0a523b841cb09a6f26ddd4d22efd98776e78d10fd996995fd00e81e08d3c25dd14a54b25a9d483677a24bbb8d1cb41a443b2c71038e6893b1b30f70758424e0f2039a48060191389033ef55", "0xa4c017311b9e930868132527a9849072b91db04fd36c619ae39c98da9e2174e6201d3c2ff1246c06b1b6815bbf3ea4a1116564f55ee2fe4c4d655e2294c0ded842cba209c255ca3d7b7f82d162f97890dfdeed087aa2f87cbfc61d61815da39d", "0x89516315a3956b455843c2555248bd94dcb19993060fe75fdd51f7aa9c9147ab13997d8a98036a8f04bee5c91d78d2990907e35a52537a8ab3ed15f1a71afdcd38044a5b6e93f662b9d36c16933a881927cacae668c4c06ee6f004c9e3989bad", "0xa1e78a011e210400c68ca76045f7da74119bff3cbe382efd2bd2ac76567c52d68d75536a91999d084043e1ce2d07d02e0b69fb99924101d2543521747536fbc51b0454aa9a4cbbec101121f597863a5c0fee2ca5eab35dff9b9085bef8b2b0d0", "0x830fd8d083e39153ecab43cabb22e29d7b44a55fba467af4ddd3f069439d2972ef53c3518de788f96b3f4f64963987d0155ba27afc28643af3de8e476ff515a68285728167408f45d99e574680bda6bacdd4322e587e4aa99386e035c0e931ad", "0xb89584da22237e3061d991b1a55a5e55dc637b8b671130d304587729348138ef87885180310efe9f9f6d3580b9d7fdcf0649e8a79d2dec8c25a9f53df0fac5d517db999029cbfdd7c2cbd3e9a5503e5d267d3d8ad752335915c92b850b14bafb", "0x959b8030733799882c5e3735479924b013756e57b893f9792bab4043e2d362d77cf308166d782e3989caa771b8a0c0a01302cb7b5e8ca12e2d6cebd59d4cd173c9dc25f438bac597fab17b4ff44997a489c168e7204b7d7c21d0938f0a2e3b51", "0xa0a9e5503d9afe0027891dab890c687fd5f5fac5741418490c64d7c15f59533dd603a50163c79402afa61bd02de486761983c94501da17e6bbe78c497f2122210071602f578adc0ebe7a4679f87fe77e09c8c122de69105f13455fea25f08e6f", "0x9811487283ad620cd7c9b303ae2f348d0e6f5ee17b504baaa817ae207adb912a00d3cc36dbf48745eb899e6b6e22f09f0f9ba29d949ecd7350fbbfe87a8c7cdd5d0e687fc807751d07634aaf7c38baf3b24a0670c38fa6ccd7431436fc95525f", "0x8a13aa5071c526e560def7d8583393942f07d88c9d8d26c98738fd65f57af2e3326dbb1edff0f39fe98eda4a13ed4fd71844254b954690154c4804e1c4a53df9dc4643f4b7b09d0860070f6b2318d0d63d28fb56bf5b6ff456a18dfc72fdfbbe", "0xb9c90ff6bff5dd97d90aee27ea1c61c1afe64b054c258b097709561fe00710e9e616773fc4bdedcbf91fbd1a6cf139bf14d20db07297418694c12c6c9b801638eeb537cb3741584a686d69532e3b6c12d8a376837f712032421987f1e770c258"], "setup_G1_lagrange": ["0x91131b2e3c1e5f0b51df8970e67080032f411571b66d301436c46f25bbfddf9ca16756430dc470bdb0d85b47fedcdbc1", "0x934d35b2a46e169915718b77127b0d4efbacdad7fdde4593af7d21d37ebcb77fe6c8dde6b8a9537854d70ef1f291a585", "0x9410ca1d0342fe7419f02194281df45e1c1ff42fd8b439de5644cc312815c21ddd2e3eeb63fb807cf837e68b76668bd5", "0xb163df7e9baeb60f69b6ee5faa538c3a564b62eb8cde6a3616083c8cb2171eedd583c9143e7e916df59bf27da5e024e8"], "roots_of_unity": [1, 3465144826073652318776269530687742778270252468765361963008, 52435875175126190479447740508185965837690552500527637822603658699938581184512, 52435875175126190475982595682112313518914282969839895044333406231173219221505]} \ No newline at end of file diff --git a/common/eth2_network_config/src/lib.rs b/common/eth2_network_config/src/lib.rs index 0d434ba741f..f760b8ed694 100644 --- a/common/eth2_network_config/src/lib.rs +++ b/common/eth2_network_config/src/lib.rs @@ -13,10 +13,11 @@ use discv5::enr::{CombinedKey, Enr}; use eth2_config::{instantiate_hardcoded_nets, HardcodedNet}; -use kzg::TrustedSetup; +use kzg::{KzgPreset, KzgPresetId, TrustedSetup}; use std::fs::{create_dir_all, File}; use std::io::{Read, Write}; use std::path::PathBuf; +use std::str::FromStr; use types::{BeaconState, ChainSpec, Config, Epoch, EthSpec, EthSpecId}; pub const DEPLOY_BLOCK_FILE: &str = "deploy_block.txt"; @@ -38,9 +39,26 @@ pub const DEFAULT_HARDCODED_NETWORK: &str = "mainnet"; /// /// This is done to ensure that testnets also inherit the high security and /// randomness of the mainnet kzg trusted setup ceremony. -pub const TRUSTED_SETUP: &[u8] = +const TRUSTED_SETUP: &[u8] = include_bytes!("../built_in_network_configs/testing_trusted_setups.json"); +const TRUSTED_SETUP_MINIMAL: &[u8] = + include_bytes!("../built_in_network_configs/minimal_testing_trusted_setups.json"); + +pub fn get_trusted_setup() -> &'static [u8] { + match P::spec_name() { + KzgPresetId::Mainnet => TRUSTED_SETUP, + KzgPresetId::Minimal => TRUSTED_SETUP_MINIMAL, + } +} + +pub fn get_trusted_setup_from_id(id: KzgPresetId) -> &'static [u8] { + match id { + KzgPresetId::Mainnet => TRUSTED_SETUP, + KzgPresetId::Minimal => TRUSTED_SETUP_MINIMAL, + } +} + /// Specifies an Eth2 network. /// /// See the crate-level documentation for more details. @@ -73,7 +91,9 @@ impl Eth2NetworkConfig { let kzg_trusted_setup = if let Some(epoch) = config.deneb_fork_epoch { // Only load the trusted setup if the deneb fork epoch is set if epoch.value != Epoch::max_value() { - let trusted_setup: TrustedSetup = serde_json::from_reader(TRUSTED_SETUP) + let trusted_setup_bytes = + get_trusted_setup_from_id(KzgPresetId::from_str(&config.preset_base)?); + let trusted_setup: TrustedSetup = serde_json::from_reader(trusted_setup_bytes) .map_err(|e| format!("Unable to read trusted setup file: {}", e))?; Some(trusted_setup) } else { @@ -239,8 +259,10 @@ impl Eth2NetworkConfig { let kzg_trusted_setup = if let Some(epoch) = config.deneb_fork_epoch { // Only load the trusted setup if the deneb fork epoch is set if epoch.value != Epoch::max_value() { - let trusted_setup: TrustedSetup = serde_json::from_reader(TRUSTED_SETUP) - .map_err(|e| format!("Unable to read trusted setup file: {}", e))?; + let trusted_setup: TrustedSetup = serde_json::from_reader( + get_trusted_setup_from_id(KzgPresetId::from_str(&config.preset_base)?), + ) + .map_err(|e| format!("Unable to read trusted setup file: {}", e))?; Some(trusted_setup) } else { None diff --git a/crypto/kzg/Cargo.toml b/crypto/kzg/Cargo.toml index f1975d5ff49..b26b1343331 100644 --- a/crypto/kzg/Cargo.toml +++ b/crypto/kzg/Cargo.toml @@ -16,8 +16,8 @@ serde_derive = "1.0.116" ethereum_serde_utils = "0.5.0" hex = "0.4.2" ethereum_hashing = "1.0.0-beta.2" -c-kzg = { git = "https://github.com/michaelsproul/c-kzg-4844", rev = "e652ce854a5f1855174cc89ce588513529656588" } -c_kzg_min = { package = "c-kzg", git = "https://github.com/michaelsproul//c-kzg-4844", rev = "e652ce854a5f1855174cc89ce588513529656588" } +c-kzg = { git = "https://github.com/realbigsean/c-kzg-4844", rev = "1f25100d889c1dcc221f0c624225b38d511387a2" , features = ["mainnet-spec"]} +c_kzg_min = { package = "c-kzg", git = "https://github.com/realbigsean//c-kzg-4844", rev = "1f25100d889c1dcc221f0c624225b38d511387a2", features = ["minimal-spec"] } arbitrary = { version = "1.0", features = ["derive"], optional = true } [features] diff --git a/crypto/kzg/src/lib.rs b/crypto/kzg/src/lib.rs index 4404e076bed..a3a6bff60ac 100644 --- a/crypto/kzg/src/lib.rs +++ b/crypto/kzg/src/lib.rs @@ -5,6 +5,7 @@ mod trusted_setup; use serde_derive::{Deserialize, Serialize}; use std::fmt::Debug; use std::ops::Deref; +use std::str::FromStr; pub use crate::{kzg_commitment::KzgCommitment, kzg_proof::KzgProof, trusted_setup::TrustedSetup}; pub use c_kzg::{Bytes32, Bytes48}; @@ -40,10 +41,27 @@ pub trait BlobTrait: Sized + Clone { fn from_bytes(bytes: &[u8]) -> Result; } +pub enum KzgPresetId { + Mainnet, + Minimal, +} + +impl FromStr for KzgPresetId { + type Err = String; + + fn from_str(s: &str) -> Result { + match s { + "mainnet" => Ok(KzgPresetId::Mainnet), + "minimal" => Ok(KzgPresetId::Minimal), + _ => Err(format!("Unknown eth spec: {}", s)), + } + } +} + pub trait KzgPreset: 'static + Default + Sync + Send + Clone + Debug + PartialEq + Eq + for<'a> arbitrary::Arbitrary<'a> { - type KzgSettings: Debug; + type KzgSettings: Debug + Sync + Send; type Blob: BlobTrait; type Bytes32: From<[u8; 32]> + Deref; type Bytes48: From + From; @@ -53,6 +71,8 @@ pub trait KzgPreset: const BYTES_PER_FIELD_ELEMENT: usize; const FIELD_ELEMENTS_PER_BLOB: usize; + fn spec_name() -> KzgPresetId; + fn bytes32_in(bytes: Bytes32) -> Self::Bytes32 { let bytes: [u8; 32] = *bytes; Self::Bytes32::from(bytes) @@ -106,7 +126,7 @@ pub trait KzgPreset: } macro_rules! implement_kzg_preset { - ($preset_type:ident, $module_name:ident) => { + ($preset_type:ident, $module_name:ident, $preset_id:ident) => { impl KzgPreset for $preset_type { type KzgSettings = $module_name::KzgSettings; type Blob = $module_name::Blob; @@ -118,6 +138,10 @@ macro_rules! implement_kzg_preset { const BYTES_PER_FIELD_ELEMENT: usize = $module_name::BYTES_PER_FIELD_ELEMENT; const FIELD_ELEMENTS_PER_BLOB: usize = $module_name::FIELD_ELEMENTS_PER_BLOB; + fn spec_name() -> KzgPresetId { + KzgPresetId::$preset_id + } + fn load_trusted_setup( trusted_setup: TrustedSetup, ) -> Result { @@ -224,8 +248,8 @@ pub struct MainnetKzgPreset; #[derive(Clone, PartialEq, Eq, Debug, Default, Serialize, Deserialize, arbitrary::Arbitrary)] pub struct MinimalKzgPreset; -implement_kzg_preset!(MainnetKzgPreset, c_kzg); -implement_kzg_preset!(MinimalKzgPreset, c_kzg_min); +implement_kzg_preset!(MainnetKzgPreset, c_kzg, Mainnet); +implement_kzg_preset!(MinimalKzgPreset, c_kzg_min, Minimal); /// A wrapper over a kzg library that holds the trusted setup parameters. #[derive(Debug)] diff --git a/lcli/src/new_testnet.rs b/lcli/src/new_testnet.rs index dfb9a283a0d..eaf963a8c07 100644 --- a/lcli/src/new_testnet.rs +++ b/lcli/src/new_testnet.rs @@ -1,7 +1,7 @@ use account_utils::eth2_keystore::keypair_from_secret; use clap::ArgMatches; use clap_utils::{parse_optional, parse_required, parse_ssz_optional}; -use eth2_network_config::{Eth2NetworkConfig, TRUSTED_SETUP}; +use eth2_network_config::{get_trusted_setup, Eth2NetworkConfig}; use eth2_wallet::bip39::Seed; use eth2_wallet::bip39::{Language, Mnemonic}; use eth2_wallet::{recover_validator_secret_from_mnemonic, KeyType}; @@ -199,8 +199,9 @@ pub fn run(testnet_dir_path: PathBuf, matches: &ArgMatches) -> Resul let kzg_trusted_setup = if let Some(epoch) = spec.deneb_fork_epoch { // Only load the trusted setup if the deneb fork epoch is set if epoch != Epoch::max_value() { - let trusted_setup: TrustedSetup = serde_json::from_reader(TRUSTED_SETUP) - .map_err(|e| format!("Unable to read trusted setup file: {}", e))?; + let trusted_setup: TrustedSetup = + serde_json::from_reader(get_trusted_setup::()) + .map_err(|e| format!("Unable to read trusted setup file: {}", e))?; Some(trusted_setup) } else { None diff --git a/testing/ef_tests/src/cases/kzg_blob_to_kzg_commitment.rs b/testing/ef_tests/src/cases/kzg_blob_to_kzg_commitment.rs index f160b8b2397..d26d57de3b5 100644 --- a/testing/ef_tests/src/cases/kzg_blob_to_kzg_commitment.rs +++ b/testing/ef_tests/src/cases/kzg_blob_to_kzg_commitment.rs @@ -31,7 +31,7 @@ impl Case for KZGBlobToKZGCommitment { } fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> { - let kzg = get_kzg()?; + let kzg = get_kzg::()?; let commitment = parse_blob::(&self.input.blob).and_then(|blob| { blob_to_kzg_commitment::(&kzg, blob).map_err(|e| { diff --git a/testing/ef_tests/src/cases/kzg_compute_blob_kzg_proof.rs b/testing/ef_tests/src/cases/kzg_compute_blob_kzg_proof.rs index 30187f91cef..598fd2e780e 100644 --- a/testing/ef_tests/src/cases/kzg_compute_blob_kzg_proof.rs +++ b/testing/ef_tests/src/cases/kzg_compute_blob_kzg_proof.rs @@ -38,7 +38,7 @@ impl Case for KZGComputeBlobKZGProof { Ok((blob, commitment)) }; - let kzg = get_kzg()?; + let kzg = get_kzg::()?; let proof = parse_input(&self.input).and_then(|(blob, commitment)| { compute_blob_kzg_proof::(&kzg, &blob, commitment) .map_err(|e| Error::InternalError(format!("Failed to compute kzg proof: {:?}", e))) diff --git a/testing/ef_tests/src/cases/kzg_compute_kzg_proof.rs b/testing/ef_tests/src/cases/kzg_compute_kzg_proof.rs index f851947d9f3..7483bed0445 100644 --- a/testing/ef_tests/src/cases/kzg_compute_kzg_proof.rs +++ b/testing/ef_tests/src/cases/kzg_compute_kzg_proof.rs @@ -45,7 +45,7 @@ impl Case for KZGComputeKZGProof { Ok((blob, z)) }; - let kzg = get_kzg()?; + let kzg = get_kzg::()?; let proof = parse_input(&self.input).and_then(|(blob, z)| { compute_kzg_proof::(&kzg, blob, z) .map_err(|e| Error::InternalError(format!("Failed to compute kzg proof: {:?}", e))) diff --git a/testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof.rs b/testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof.rs index fdc68a59201..11bb2047828 100644 --- a/testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof.rs +++ b/testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof.rs @@ -1,15 +1,15 @@ use super::*; use crate::case_result::compare_result; use beacon_chain::kzg_utils::validate_blob; -use eth2_network_config::TRUSTED_SETUP; -use kzg::{Kzg, KzgCommitment, KzgProof, TrustedSetup}; +use eth2_network_config::get_trusted_setup; +use kzg::{Kzg, KzgCommitment, KzgPreset, KzgProof, TrustedSetup}; use serde_derive::Deserialize; use std::convert::TryInto; use std::marker::PhantomData; use types::Blob; -pub fn get_kzg() -> Result { - let trusted_setup: TrustedSetup = serde_json::from_reader(TRUSTED_SETUP) +pub fn get_kzg() -> Result, Error> { + let trusted_setup: TrustedSetup = serde_json::from_reader(get_trusted_setup::

()) .map_err(|e| Error::InternalError(format!("Failed to initialize kzg: {:?}", e)))?; Kzg::new_from_trusted_setup(trusted_setup) .map_err(|e| Error::InternalError(format!("Failed to initialize kzg: {:?}", e))) @@ -81,7 +81,7 @@ impl Case for KZGVerifyBlobKZGProof { Ok((blob, commitment, proof)) }; - let kzg = get_kzg()?; + let kzg = get_kzg::()?; let result = parse_input(&self.input).and_then(|(blob, commitment, proof)| { validate_blob::(&kzg, blob, commitment, proof) .map_err(|e| Error::InternalError(format!("Failed to validate blob: {:?}", e))) diff --git a/testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof_batch.rs b/testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof_batch.rs index 960ad4e4f2d..90dc1614b4f 100644 --- a/testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof_batch.rs +++ b/testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof_batch.rs @@ -51,7 +51,7 @@ impl Case for KZGVerifyBlobKZGProofBatch { Ok((commitments, blobs, proofs)) }; - let kzg = get_kzg()?; + let kzg = get_kzg::()?; let result = parse_input(&self.input).and_then(|(commitments, blobs, proofs)| { validate_blobs::(&kzg, &commitments, &blobs, &proofs) .map_err(|e| Error::InternalError(format!("Failed to validate blobs: {:?}", e))) diff --git a/testing/ef_tests/src/cases/kzg_verify_kzg_proof.rs b/testing/ef_tests/src/cases/kzg_verify_kzg_proof.rs index 638c3b28359..a6acbec18ae 100644 --- a/testing/ef_tests/src/cases/kzg_verify_kzg_proof.rs +++ b/testing/ef_tests/src/cases/kzg_verify_kzg_proof.rs @@ -41,7 +41,7 @@ impl Case for KZGVerifyKZGProof { Ok((commitment, z, y, proof)) }; - let kzg = get_kzg()?; + let kzg = get_kzg::()?; let result = parse_input(&self.input).and_then(|(commitment, z, y, proof)| { verify_kzg_proof::(&kzg, commitment, proof, z, y) .map_err(|e| Error::InternalError(format!("Failed to validate proof: {:?}", e))) From 18af05a054053e9be0e183da963be36d03f98644 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Tue, 27 Jun 2023 16:48:36 -0400 Subject: [PATCH 4/4] fix remaining tests --- .../beacon_chain/src/blob_verification.rs | 82 +++++++++++++------ beacon_node/beacon_chain/src/test_utils.rs | 29 +++---- .../beacon_chain/tests/block_verification.rs | 41 ++++++++-- beacon_node/beacon_chain/tests/store_tests.rs | 12 ++- beacon_node/beacon_chain/tests/tests.rs | 2 +- 5 files changed, 118 insertions(+), 48 deletions(-) diff --git a/beacon_node/beacon_chain/src/blob_verification.rs b/beacon_node/beacon_chain/src/blob_verification.rs index 8ae25551ddd..8e7c4dc0746 100644 --- a/beacon_node/beacon_chain/src/blob_verification.rs +++ b/beacon_node/beacon_chain/src/blob_verification.rs @@ -4,8 +4,8 @@ use state_processing::state_advance::partial_state_advance; use std::sync::Arc; use crate::beacon_chain::{ - BeaconChain, BeaconChainTypes, MAXIMUM_GOSSIP_CLOCK_DISPARITY, - VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT, + BeaconChain, BeaconChainTypes, BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT, + MAXIMUM_GOSSIP_CLOCK_DISPARITY, VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT, }; use crate::data_availability_checker::{ AvailabilityCheckError, AvailabilityPendingBlock, AvailableBlock, @@ -240,36 +240,72 @@ pub fn validate_blob_sidecar_for_gossip( "block_root" => %block_root, "index" => %blob_index, ); - // The cached head state is in the same epoch as the blob or the state has already been - // advanced to the blob's epoch - let snapshot = &chain.canonical_head.cached_head().snapshot; - if snapshot.beacon_state.current_epoch() == blob_slot.epoch(T::EthSpec::slots_per_epoch()) { - ( - snapshot - .beacon_state - .get_beacon_proposer_index(blob_slot, &chain.spec)?, - snapshot.beacon_state.fork(), - ) + if let Some(mut snapshot) = chain + .snapshot_cache + .try_read_for(BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT) + .and_then(|snapshot_cache| { + snapshot_cache.get_cloned(block_parent_root, CloneConfig::committee_caches_only()) + }) + { + if snapshot.beacon_state.slot() == blob_slot { + debug!( + chain.log, + "Cloning snapshot cache state for blob verification"; + "block_root" => %block_root, + "index" => %blob_index, + ); + ( + snapshot + .beacon_state + .get_beacon_proposer_index(blob_slot, &chain.spec)?, + snapshot.beacon_state.fork(), + ) + } else { + debug!( + chain.log, + "Cloning and advancing snapshot cache state for blob verification"; + "block_root" => %block_root, + "index" => %blob_index, + ); + let state = cheap_state_advance_to_obtain_committees( + &mut snapshot.beacon_state, + Some(snapshot.beacon_block_root), + blob_slot, + &chain.spec, + )?; + ( + state.get_beacon_proposer_index(blob_slot, &chain.spec)?, + state.fork(), + ) + } } // Need to advance the state to get the proposer index else { - // Reaching this condition too often might be an issue since we could theoretically have - // 5 threads (4 blob indices + 1 block) cloning the state. - // We shouldn't be seeing this condition a lot because we try to advance the state - // 3 seconds before the start of a slot. However, if this becomes an issue during testing, we should - // consider sending a blob for reprocessing to reduce the number of state clones. warn!( chain.log, - "Cached head not advanced for blob verification"; + "Snapshot cache miss for blob verification"; "block_root" => %block_root, "index" => %blob_index, - "action" => "contact the devs if you see this msg too often" ); - // The state produced is only valid for determining proposer/attester shuffling indices. - let mut cloned_state = snapshot.clone_with(CloneConfig::committee_caches_only()); + + let parent_block = chain + .get_blinded_block(&block_parent_root) + .map_err(BlobError::BeaconChainError)? + .ok_or_else(|| { + BlobError::from(BeaconChainError::MissingBeaconBlock(block_parent_root)) + })?; + + let mut parent_state = chain + .get_state(&parent_block.state_root(), Some(parent_block.slot()))? + .ok_or_else(|| { + BeaconChainError::DBInconsistent(format!( + "Missing state {:?}", + parent_block.state_root() + )) + })?; let state = cheap_state_advance_to_obtain_committees( - &mut cloned_state.beacon_state, - None, + &mut parent_state, + Some(parent_block.state_root()), blob_slot, &chain.spec, )?; diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index bf7abdb1a61..b84f94f96d2 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -745,6 +745,20 @@ where .execution_block_generator() } + pub fn get_head_block(&self) -> BlockWrapper { + let block = self.chain.head_beacon_block(); + let block_root = block.canonical_root(); + let blobs = self.chain.get_blobs(&block_root).unwrap(); + BlockWrapper::new(block, blobs) + } + + pub fn get_full_block(&self, block_root: &Hash256) -> BlockWrapper { + let block = self.chain.get_blinded_block(&block_root).unwrap().unwrap(); + let full_block = self.chain.store.make_full_block(block_root, block).unwrap(); + let blobs = self.chain.get_blobs(&block_root).unwrap(); + BlockWrapper::new(Arc::new(full_block), blobs) + } + pub fn get_all_validators(&self) -> Vec { (0..self.validator_keypairs.len()).collect() } @@ -909,20 +923,7 @@ where let signed_blobs: SignedBlobSidecarList = Vec::from(blobs) .into_iter() .map(|blob| { - // Need to overwrite the proposer index before signing, not sure of a - // better way to do this. - let new_blob = BlobSidecar { - block_root: blob.block_root, - index: blob.index, - slot: blob.slot, - block_parent_root: blob.block_parent_root, - proposer_index: proposer_index as u64, - blob: blob.blob.clone(), - kzg_commitment: blob.kzg_commitment, - kzg_proof: blob.kzg_proof, - }; - let new_blob = Arc::new(new_blob); - new_blob.sign( + blob.sign( &self.validator_keypairs[proposer_index].sk, &state.fork(), state.genesis_validators_root(), diff --git a/beacon_node/beacon_chain/tests/block_verification.rs b/beacon_node/beacon_chain/tests/block_verification.rs index 4b93a988864..57692c4e5d0 100644 --- a/beacon_node/beacon_chain/tests/block_verification.rs +++ b/beacon_node/beacon_chain/tests/block_verification.rs @@ -411,6 +411,7 @@ async fn chain_segment_non_linear_slots() { async fn assert_invalid_signature( chain_segment: &[BeaconSnapshot], + chain_segment_blobs: &[Option>], harness: &BeaconChainHarness>, block_index: usize, snapshots: &[BeaconSnapshot], @@ -418,7 +419,8 @@ async fn assert_invalid_signature( ) { let blocks: Vec> = snapshots .iter() - .map(|snapshot| snapshot.beacon_block.clone().into()) + .zip(chain_segment_blobs.iter()) + .map(|(snapshot, blobs)| BlockWrapper::new(snapshot.beacon_block.clone(), blobs.clone())) .collect(); // Ensure the block will be rejected if imported in a chain segment. @@ -442,7 +444,8 @@ async fn assert_invalid_signature( let ancestor_blocks = chain_segment .iter() .take(block_index) - .map(|snapshot| snapshot.beacon_block.clone().into()) + .zip(chain_segment_blobs.iter()) + .map(|(snapshot, blobs)| BlockWrapper::new(snapshot.beacon_block.clone(), blobs.clone())) .collect(); // We don't care if this fails, we just call this to ensure that all prior blocks have been // imported prior to this test. @@ -460,7 +463,10 @@ async fn assert_invalid_signature( .chain .process_block( snapshots[block_index].beacon_block.canonical_root(), - snapshots[block_index].beacon_block.clone(), + BlockWrapper::new( + snapshots[block_index].beacon_block.clone(), + chain_segment_blobs[block_index].clone(), + ), CountUnrealized::True, NotifyExecutionLayer::Yes, ) @@ -510,7 +516,10 @@ async fn invalid_signature_gossip_block() { let ancestor_blocks = chain_segment .iter() .take(block_index) - .map(|snapshot| snapshot.beacon_block.clone().into()) + .zip(chain_segment_blobs.iter()) + .map(|(snapshot, blobs)| { + BlockWrapper::new(snapshot.beacon_block.clone(), blobs.clone()) + }) .collect(); harness .chain @@ -558,7 +567,10 @@ async fn invalid_signature_block_proposal() { )); let blocks: Vec> = snapshots .iter() - .map(|snapshot| snapshot.beacon_block.clone().into()) + .zip(chain_segment_blobs.iter()) + .map(|(snapshot, blobs)| { + BlockWrapper::new(snapshot.beacon_block.clone(), blobs.clone()) + }) .collect::>(); // Ensure the block will be rejected if imported in a chain segment. assert!( @@ -591,7 +603,15 @@ async fn invalid_signature_randao_reveal() { Arc::new(SignedBeaconBlock::from_block(block, signature)); update_parent_roots(&mut snapshots); update_proposal_signatures(&mut snapshots, &harness); - assert_invalid_signature(&chain_segment, &harness, block_index, &snapshots, "randao").await; + assert_invalid_signature( + &chain_segment, + &chain_segment_blobs, + &harness, + block_index, + &snapshots, + "randao", + ) + .await; } } @@ -627,6 +647,7 @@ async fn invalid_signature_proposer_slashing() { update_proposal_signatures(&mut snapshots, &harness); assert_invalid_signature( &chain_segment, + &chain_segment_blobs, &harness, block_index, &snapshots, @@ -679,6 +700,7 @@ async fn invalid_signature_attester_slashing() { update_proposal_signatures(&mut snapshots, &harness); assert_invalid_signature( &chain_segment, + &chain_segment_blobs, &harness, block_index, &snapshots, @@ -709,6 +731,7 @@ async fn invalid_signature_attestation() { update_proposal_signatures(&mut snapshots, &harness); assert_invalid_signature( &chain_segment, + &chain_segment_blobs, &harness, block_index, &snapshots, @@ -757,7 +780,10 @@ async fn invalid_signature_deposit() { update_proposal_signatures(&mut snapshots, &harness); let blocks: Vec> = snapshots .iter() - .map(|snapshot| snapshot.beacon_block.clone().into()) + .zip(chain_segment_blobs.iter()) + .map(|(snapshot, blobs)| { + BlockWrapper::new(snapshot.beacon_block.clone(), blobs.clone()) + }) .collect(); assert!( !matches!( @@ -802,6 +828,7 @@ async fn invalid_signature_exit() { update_proposal_signatures(&mut snapshots, &harness); assert_invalid_signature( &chain_segment, + &chain_segment_blobs, &harness, block_index, &snapshots, diff --git a/beacon_node/beacon_chain/tests/store_tests.rs b/beacon_node/beacon_chain/tests/store_tests.rs index 81378e76752..76a66895123 100644 --- a/beacon_node/beacon_chain/tests/store_tests.rs +++ b/beacon_node/beacon_chain/tests/store_tests.rs @@ -1,6 +1,7 @@ #![cfg(not(debug_assertions))] use beacon_chain::attestation_verification::Error as AttnError; +use beacon_chain::blob_verification::BlockWrapper; use beacon_chain::builder::BeaconChainBuilder; use beacon_chain::schema_change::migrate_schema; use beacon_chain::test_utils::{ @@ -2178,12 +2179,14 @@ async fn weak_subjectivity_sync() { assert_eq!(new_blocks[0].beacon_block.slot(), wss_slot + 1); for snapshot in new_blocks { + let block_root = snapshot.beacon_block_root; let full_block = harness .chain .get_block(&snapshot.beacon_block_root) .await .unwrap() .unwrap(); + let blobs = harness.chain.get_blobs(&block_root).expect("blobs"); let slot = full_block.slot(); let state_root = full_block.state_root(); @@ -2191,7 +2194,7 @@ async fn weak_subjectivity_sync() { beacon_chain .process_block( full_block.canonical_root(), - Arc::new(full_block), + BlockWrapper::new(Arc::new(full_block), blobs), CountUnrealized::True, NotifyExecutionLayer::Yes, ) @@ -2239,16 +2242,19 @@ async fn weak_subjectivity_sync() { let mut available_blocks = vec![]; for blinded in historical_blocks { + let block_root = blinded.canonical_root(); let full_block = harness .chain - .get_block(&blinded.canonical_root()) + .get_block(&block_root) .await .expect("should get block") .expect("should get block"); + let blobs = harness.chain.get_blobs(&block_root).expect("blobs"); + if let MaybeAvailableBlock::Available(block) = harness .chain .data_availability_checker - .check_availability(full_block.into()) + .check_availability(BlockWrapper::new(Arc::new(full_block), blobs)) .expect("should check availability") { available_blocks.push(block); diff --git a/beacon_node/beacon_chain/tests/tests.rs b/beacon_node/beacon_chain/tests/tests.rs index f73cd0acad4..355c838cc2d 100644 --- a/beacon_node/beacon_chain/tests/tests.rs +++ b/beacon_node/beacon_chain/tests/tests.rs @@ -685,7 +685,7 @@ async fn run_skip_slot_test(skip_slots: u64) { .chain .process_block( harness_a.chain.head_snapshot().beacon_block_root, - harness_a.chain.head_snapshot().beacon_block.clone(), + harness_a.get_head_block(), CountUnrealized::True, NotifyExecutionLayer::Yes, )