From 2a83d1099de31e990a30aa600a2f6e8e5a3a4656 Mon Sep 17 00:00:00 2001 From: Kevaundray Wedderburn Date: Tue, 23 Jul 2024 08:56:05 +0100 Subject: [PATCH 1/5] chore: return vectors fo cells and proofs --- eip7594/src/prover.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/eip7594/src/prover.rs b/eip7594/src/prover.rs index 8dadc7a2..33c3ba96 100644 --- a/eip7594/src/prover.rs +++ b/eip7594/src/prover.rs @@ -4,16 +4,13 @@ use kzg_multi_open::{ }; use crate::{ - constants::{ - CELLS_PER_EXT_BLOB, FIELD_ELEMENTS_PER_BLOB, FIELD_ELEMENTS_PER_CELL, - FIELD_ELEMENTS_PER_EXT_BLOB, - }, + constants::{FIELD_ELEMENTS_PER_BLOB, FIELD_ELEMENTS_PER_CELL, FIELD_ELEMENTS_PER_EXT_BLOB}, errors::Error, serialization::{ deserialize_blob_to_scalars, serialize_cells_and_proofs, serialize_g1_compressed, }, trusted_setup::TrustedSetup, - BlobRef, Cell, CellIndex, CellRef, KZGCommitment, KZGProof, DASContext, + BlobRef, Cell, CellIndex, CellRef, DASContext, KZGCommitment, KZGProof, }; /// Context object that is used to call functions in the prover API. @@ -79,7 +76,7 @@ impl DASContext { pub fn compute_cells_and_kzg_proofs( &self, blob: BlobRef, - ) -> Result<([Cell; CELLS_PER_EXT_BLOB], [KZGProof; CELLS_PER_EXT_BLOB]), Error> { + ) -> Result<(Vec, Vec), Error> { self.thread_pool.install(|| { // Deserialization // @@ -107,7 +104,7 @@ impl DASContext { &self, cell_indices: Vec, cells: Vec, - ) -> Result<([Cell; CELLS_PER_EXT_BLOB], [KZGProof; CELLS_PER_EXT_BLOB]), Error> { + ) -> Result<(Vec, Vec), Error> { self.thread_pool.install(|| { // Recover polynomial // From 3ad52d482eefc2d322a0897385138a718385765b Mon Sep 17 00:00:00 2001 From: Kevaundray Wedderburn Date: Tue, 23 Jul 2024 08:56:43 +0100 Subject: [PATCH 2/5] refactor parts of code that rely on Cells and Proofs being arrays --- bindings/c/src/compute_cells_and_kzg_proofs.rs | 6 +++--- bindings/c/src/recover_cells_and_kzg_proofs.rs | 12 +++++++++--- bindings/java/rust_code/src/lib.rs | 8 ++++---- eip7594/src/serialization.rs | 17 +++++++++-------- 4 files changed, 25 insertions(+), 18 deletions(-) diff --git a/bindings/c/src/compute_cells_and_kzg_proofs.rs b/bindings/c/src/compute_cells_and_kzg_proofs.rs index 51a42834..d93ae883 100644 --- a/bindings/c/src/compute_cells_and_kzg_proofs.rs +++ b/bindings/c/src/compute_cells_and_kzg_proofs.rs @@ -20,11 +20,11 @@ pub(crate) fn _compute_cells_and_kzg_proofs( let (cells, proofs) = ctx .compute_cells_and_kzg_proofs(blob) .map_err(|err| CResult::with_error(&format!("{:?}", err)))?; - let cells_unboxed = cells.map(|cell| cell.to_vec()); + let cells_unboxed: Vec<_> = cells.into_iter().map(|cell| cell.to_vec()).collect(); // Write to output - write_to_2d_slice::<_, CELLS_PER_EXT_BLOB>(out_cells, cells_unboxed); - write_to_2d_slice::<_, CELLS_PER_EXT_BLOB>(out_proofs, proofs); + write_to_2d_slice::<_, CELLS_PER_EXT_BLOB>(out_cells, cells_unboxed.try_into().unwrap()); + write_to_2d_slice::<_, CELLS_PER_EXT_BLOB>(out_proofs, proofs.try_into().unwrap()); Ok(()) } diff --git a/bindings/c/src/recover_cells_and_kzg_proofs.rs b/bindings/c/src/recover_cells_and_kzg_proofs.rs index 7fd60a66..70464a51 100644 --- a/bindings/c/src/recover_cells_and_kzg_proofs.rs +++ b/bindings/c/src/recover_cells_and_kzg_proofs.rs @@ -26,11 +26,17 @@ pub(crate) fn _recover_cells_and_proofs( let (recovered_cells, recovered_proofs) = ctx .recover_cells_and_proofs(cell_indices.to_vec(), cells) .map_err(|err| CResult::with_error(&format!("{:?}", err)))?; - let recovered_cells_unboxed = recovered_cells.map(|cell| cell.to_vec()); + let recovered_cells_unboxed: Vec<_> = recovered_cells + .into_iter() + .map(|cell| cell.to_vec()) + .collect(); // Write to output - write_to_2d_slice::<_, CELLS_PER_EXT_BLOB>(out_cells, recovered_cells_unboxed); - write_to_2d_slice::<_, CELLS_PER_EXT_BLOB>(out_proofs, recovered_proofs); + write_to_2d_slice::<_, CELLS_PER_EXT_BLOB>( + out_cells, + recovered_cells_unboxed.try_into().unwrap(), + ); + write_to_2d_slice::<_, CELLS_PER_EXT_BLOB>(out_proofs, recovered_proofs.try_into().unwrap()); Ok(()) } diff --git a/bindings/java/rust_code/src/lib.rs b/bindings/java/rust_code/src/lib.rs index 65c01a45..a513bde1 100644 --- a/bindings/java/rust_code/src/lib.rs +++ b/bindings/java/rust_code/src/lib.rs @@ -51,8 +51,8 @@ fn compute_cells_and_kzg_proofs<'local>( let blob = slice_to_array_ref(&blob, "blob")?; let (cells, proofs) = ctx.inner().compute_cells_and_kzg_proofs(blob)?; - let cells = cells.map(|cell| *cell); - cells_and_proofs_to_jobject(env, &cells, &proofs).map_err(Error::from) + let cells: Vec<_> = cells.into_iter().map(|cell| *cell).collect(); + cells_and_proofs_to_jobject(env, &cells, &*proofs).map_err(Error::from) } #[no_mangle] @@ -186,8 +186,8 @@ fn recover_cells_and_kzg_proofs<'local>( let (recovered_cells, recovered_proofs) = ctx.inner().recover_cells_and_proofs(cell_ids, cells)?; - let recovered_cells = recovered_cells.map(|cell| *cell); - cells_and_proofs_to_jobject(env, &recovered_cells, &recovered_proofs).map_err(Error::from) + let recovered_cells: Vec<_> = recovered_cells.into_iter().map(|cell| *cell).collect(); + cells_and_proofs_to_jobject(env, &recovered_cells, &*recovered_proofs).map_err(Error::from) } /// Converts a JLongArray to a Vec diff --git a/eip7594/src/serialization.rs b/eip7594/src/serialization.rs index 7450d21d..cdb87081 100644 --- a/eip7594/src/serialization.rs +++ b/eip7594/src/serialization.rs @@ -113,7 +113,7 @@ pub(crate) fn deserialize_cells( /// Converts a set of scalars (evaluations) to the `Cell` type. pub(crate) fn coset_evaluations_to_cells>( evaluations: impl Iterator, -) -> [Cell; CELLS_PER_EXT_BLOB] { +) -> Vec { let cells: Vec = evaluations .map(|eval| serialize_scalars_to_cell(eval.as_ref())) .map(|cell| { @@ -122,24 +122,25 @@ pub(crate) fn coset_evaluations_to_cells>( .expect("infallible: Vec should have length equal to BYTES_PER_CELL") }) .collect(); - cells - .try_into() - .unwrap_or_else(|_| panic!("expected {} number of cells", CELLS_PER_EXT_BLOB)) + // cells + // .try_into() + // .unwrap_or_else(|_| panic!("expected {} number of cells", CELLS_PER_EXT_BLOB)) } pub(crate) fn serialize_cells_and_proofs( coset_evaluations: Vec>, proofs: Vec, -) -> ([Cell; CELLS_PER_EXT_BLOB], [KZGProof; CELLS_PER_EXT_BLOB]) { +) -> (Vec, Vec) { // Serialize the evaluation sets into `Cell`s. let cells = coset_evaluations_to_cells(coset_evaluations.into_iter()); // Serialize the proofs into `KZGProof` objects. let proofs: Vec<_> = proofs.iter().map(serialize_g1_compressed).collect(); - let proofs: [KZGProof; CELLS_PER_EXT_BLOB] = proofs - .try_into() - .unwrap_or_else(|_| panic!("expected {} number of proofs", CELLS_PER_EXT_BLOB)); + // let proofs: Box<[KZGProof; CELLS_PER_EXT_BLOB]> = proofs + // .into_boxed_slice() + // .try_into() + // .unwrap_or_else(|_| panic!("expected {} number of proofs", CELLS_PER_EXT_BLOB)); (cells, proofs) } From 5479b2928e9c34ee89ce828c7dd02372ebea3579 Mon Sep 17 00:00:00 2001 From: Kevaundray Wedderburn Date: Tue, 23 Jul 2024 08:57:03 +0100 Subject: [PATCH 3/5] chore: add stackoverflow test --- eip7594/src/lib.rs | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/eip7594/src/lib.rs b/eip7594/src/lib.rs index c946020a..790902f4 100644 --- a/eip7594/src/lib.rs +++ b/eip7594/src/lib.rs @@ -68,3 +68,44 @@ impl DASContext { &self.verifier_ctx } } + +#[cfg(test)] +mod tests { + use super::*; + use bls12_381::Scalar; + #[test] + fn stack_overflow_poc() { + use rayon::prelude::*; + + fn compute_blob(offset: u64) -> Vec { + let poly: Vec<_> = (0..4096).map(|i| -Scalar::from(i + offset)).collect(); + let blob: Vec<_> = poly + .into_iter() + .flat_map(|scalar| scalar.to_bytes_be()) + .collect(); + blob + } + + const NUM_BLOBS: u64 = 100; + let blobs = (0..NUM_BLOBS).map(compute_blob).collect::>(); + + let trusted_setup = TrustedSetup::default(); + + let ctx = DASContext::with_threads(&trusted_setup, 8); + let blob_cells_and_proofs_vec: Vec<_> = blobs + .par_iter() + .map(|blob_vec| { + let mut blob = [0; BYTES_PER_BLOB]; + blob.copy_from_slice(&blob_vec); + let cells_and_proofs = ctx.compute_cells_and_kzg_proofs(&blob); + + // let cells_and_proofs = + // ctx.compute_cells_and_kzg_proofs(blob_vec.as_slice().try_into().unwrap()); + + cells_and_proofs + }) + .collect(); + + std::hint::black_box(blob_cells_and_proofs_vec); + } +} From 2d58359652c7223a328d3995272cb69b73b46139 Mon Sep 17 00:00:00 2001 From: Kevaundray Wedderburn Date: Tue, 23 Jul 2024 08:58:36 +0100 Subject: [PATCH 4/5] add workflow to test edge case in release mode --- ...t-rust-stackoverflow-test-release-mode.yml | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/test-rust-stackoverflow-test-release-mode.yml diff --git a/.github/workflows/test-rust-stackoverflow-test-release-mode.yml b/.github/workflows/test-rust-stackoverflow-test-release-mode.yml new file mode 100644 index 00000000..b28205cb --- /dev/null +++ b/.github/workflows/test-rust-stackoverflow-test-release-mode.yml @@ -0,0 +1,23 @@ +name: Test Stackoverflow test case + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + CARGO_TERM_COLOR: always + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Run tests + run: cargo test --release stack_overflow_poc -- --show-output From 1d2b2f7db4b3bf9ee54f8e2a8beca1a5026131a6 Mon Sep 17 00:00:00 2001 From: Kevaundray Wedderburn Date: Tue, 23 Jul 2024 09:05:04 +0100 Subject: [PATCH 5/5] remove allocation in map function --- eip7594/src/lib.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/eip7594/src/lib.rs b/eip7594/src/lib.rs index 790902f4..bd962354 100644 --- a/eip7594/src/lib.rs +++ b/eip7594/src/lib.rs @@ -95,12 +95,12 @@ mod tests { let blob_cells_and_proofs_vec: Vec<_> = blobs .par_iter() .map(|blob_vec| { - let mut blob = [0; BYTES_PER_BLOB]; - blob.copy_from_slice(&blob_vec); - let cells_and_proofs = ctx.compute_cells_and_kzg_proofs(&blob); + // let mut blob = [0; BYTES_PER_BLOB]; + // blob.copy_from_slice(&blob_vec); + // let cells_and_proofs = ctx.compute_cells_and_kzg_proofs(&blob); - // let cells_and_proofs = - // ctx.compute_cells_and_kzg_proofs(blob_vec.as_slice().try_into().unwrap()); + let cells_and_proofs = + ctx.compute_cells_and_kzg_proofs(blob_vec.as_slice().try_into().unwrap()); cells_and_proofs })