From 9f7c525a8e9d44b42425405faf3b426c84e946f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20=C4=8Cerni=C4=8D?= Date: Thu, 3 Oct 2024 11:02:57 +0200 Subject: [PATCH 01/24] feat: commd --- .vscode/settings.json | 18 +- Cargo.lock | 11 +- Cargo.toml | 2 + cli/polka-storage-provider/Cargo.toml | 2 +- .../src/commands/utils/commp.rs | 94 ++---- .../src/commands/utils/mod.rs | 23 +- pallets/market/Cargo.toml | 2 + pallets/market/src/commd/mod.rs | 290 ++++++++++++++++++ pallets/market/src/commd/zero.rs | 189 ++++++++++++ pallets/market/src/lib.rs | 35 ++- primitives/proofs/src/types.rs | 8 + primitives/shared/Cargo.toml | 18 ++ primitives/shared/src/commcid.rs | 88 ++++++ primitives/shared/src/lib.rs | 8 + primitives/shared/src/piece.rs | 136 ++++++++ 15 files changed, 827 insertions(+), 97 deletions(-) create mode 100644 pallets/market/src/commd/mod.rs create mode 100644 pallets/market/src/commd/zero.rs create mode 100644 primitives/shared/Cargo.toml create mode 100644 primitives/shared/src/commcid.rs create mode 100644 primitives/shared/src/lib.rs create mode 100644 primitives/shared/src/piece.rs diff --git a/.vscode/settings.json b/.vscode/settings.json index eddc5902d..33cd79152 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,11 +1,11 @@ { - "files.trimTrailingWhitespace": true, - "rust-analyzer.rustfmt.extraArgs": ["+nightly"], - "rust-analyzer.checkOnSave": false, - "coverage-gutters.coverageBaseDir": "coverage", - "coverage-gutters.coverageFileNames": [ - "pallet-storage-provider/lcov.info", - "pallet-market/lcov.info", - "mater/lcov.info" - ] + "files.trimTrailingWhitespace": true, + "rust-analyzer.rustfmt.extraArgs": ["+nightly"], + "rust-analyzer.checkOnSave": false, + "coverage-gutters.coverageBaseDir": "coverage", + "coverage-gutters.coverageFileNames": [ + "pallet-storage-provider/lcov.info", + "pallet-market/lcov.info", + "mater/lcov.info" + ] } diff --git a/Cargo.lock b/Cargo.lock index 96f5596c5..fd0c2036f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8215,7 +8215,9 @@ dependencies = [ "pallet-storage-provider", "parity-scale-codec", "primitives-proofs", + "primitives-shared", "scale-info", + "sha2 0.10.8", "sp-arithmetic 26.0.0", "sp-core 34.0.0", "sp-io 38.0.0", @@ -9390,12 +9392,12 @@ dependencies = [ "filecoin-hashers", "fr32", "futures", - "ipld-core", "jsonrpsee 0.23.2", "mater", "parity-scale-codec", "polka-storage-proofs", "primitives-proofs", + "primitives-shared", "sc-cli", "serde", "serde_json", @@ -10858,6 +10860,13 @@ dependencies = [ "sp-runtime 39.0.0", ] +[[package]] +name = "primitives-shared" +version = "0.1.0" +dependencies = [ + "cid 0.11.1", +] + [[package]] name = "prioritized-metered-channel" version = "0.6.1" diff --git a/Cargo.toml b/Cargo.toml index d6302f03b..3b29b0e59 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ members = [ "pallets/proofs", "pallets/storage-provider", "primitives/proofs", + "primitives/shared", "runtime", "storage/mater", "storage/mater-cli", @@ -126,6 +127,7 @@ pallet-storage-provider = { path = "pallets/storage-provider", default-features polka-storage-proofs = { path = "lib/polka-storage-proofs", default-features = false } polka-storage-runtime = { path = "runtime" } primitives-proofs = { path = "primitives/proofs", default-features = false } +primitives-shared = { path = "primitives/shared" } storagext = { path = "cli/polka-storage/storagext" } # FileCoin proofs diff --git a/cli/polka-storage-provider/Cargo.toml b/cli/polka-storage-provider/Cargo.toml index e5e5df7d3..e0774e1b9 100644 --- a/cli/polka-storage-provider/Cargo.toml +++ b/cli/polka-storage-provider/Cargo.toml @@ -18,10 +18,10 @@ codec = { workspace = true } filecoin-hashers.workspace = true fr32.workspace = true futures = { workspace = true } -ipld-core.workspace = true jsonrpsee = { workspace = true, features = ["http-client", "macros", "server", "ws-client"] } polka-storage-proofs = { workspace = true, features = ["std", "substrate"] } primitives-proofs = { workspace = true, features = ["std"] } +primitives-shared = { workspace = true } sc-cli = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/cli/polka-storage-provider/src/commands/utils/commp.rs b/cli/polka-storage-provider/src/commands/utils/commp.rs index 7fa9e2cc3..337e4b8c0 100644 --- a/cli/polka-storage-provider/src/commands/utils/commp.rs +++ b/cli/polka-storage-provider/src/commands/utils/commp.rs @@ -1,28 +1,16 @@ -use std::io::{BufReader, Read}; +use std::io::Read; use filecoin_hashers::{ sha256::{Sha256Domain, Sha256Hasher}, Domain, }; -use fr32::{to_padded_bytes, Fr32Reader}; -use ipld_core::cid::multihash::Multihash; -use mater::Cid; -use storage_proofs_core::{merkle::BinaryMerkleTree, util::NODE_SIZE}; +use fr32::Fr32Reader; +use primitives_shared::{commcid::Commitment, piece::PaddedPieceSize, NODE_SIZE}; +use storage_proofs_core::merkle::BinaryMerkleTree; use thiserror::Error; -/// SHA2-256 with the two most significant bits from the last byte zeroed (as -/// via a mask with 0b00111111) - used for proving trees as in Filecoin. -/// -/// https://github.com/multiformats/multicodec/blob/badcfe56bb7e0bbb06b60d57565186cd6be1f932/table.csv#L153 -pub const SHA2_256_TRUNC254_PADDED: u64 = 0x1012; - -/// Filecoin piece or sector data commitment merkle node/root (CommP & CommD) -/// -/// https://github.com/multiformats/multicodec/blob/badcfe56bb7e0bbb06b60d57565186cd6be1f932/table.csv#L554 -pub const FIL_COMMITMENT_UNSEALED: u64 = 0xf101; - /// Reader that returns zeros if the inner reader is empty. -struct ZeroPaddingReader { +pub struct ZeroPaddingReader { /// The inner reader to read from. inner: R, /// The number of bytes this 0-padding reader has left to produce. @@ -30,7 +18,7 @@ struct ZeroPaddingReader { } impl ZeroPaddingReader { - fn new(inner: R, total_size: usize) -> Self { + pub fn new(inner: R, total_size: usize) -> Self { Self { inner, remaining: total_size, @@ -63,53 +51,25 @@ impl Read for ZeroPaddingReader { } } -// Ensure that the padded piece size is valid. -fn ensure_piece_size(padded_piece_size: usize) -> Result<(), CommPError> { - if padded_piece_size < NODE_SIZE { - return Err(CommPError::PieceTooSmall); - } - - if padded_piece_size % NODE_SIZE != 0 { - return Err(CommPError::InvalidPieceSize(format!( - "padded_piece_size is not multiple of {NODE_SIZE}" - ))); - } - - Ok(()) -} - /// Calculate the piece commitment for a given data source. /// /// Reference — pub fn calculate_piece_commitment( source: R, - unpadded_piece_size: u64, -) -> Result<[u8; 32], CommPError> { - // Wrap the source in a BufReader for efficient reading. - let source = BufReader::new(source); + piece_size: PaddedPieceSize, +) -> Result { // This reader adds two zero bits to each 254 bits of data read from the source. - let fr32_reader = Fr32Reader::new(source); - // This is the padded piece size after we add 2 zero bits to each 254 bits of data. - let padded_piece_size = to_padded_bytes(unpadded_piece_size as usize); - // Final padded piece size should be 2^n where n is a positive integer. That - // is because we are using MerkleTree to calculate the piece commitment. - let padded_piece_size = padded_piece_size.next_power_of_two(); - - // Ensure that the piece size is valid, before generating a MerkeTree. - ensure_piece_size(padded_piece_size)?; - - // The reader that pads the source with zeros - let mut zero_padding_reader = ZeroPaddingReader::new(fr32_reader, padded_piece_size); + let mut fr32_reader = Fr32Reader::new(source); // Buffer used for reading data used for leafs. - let mut buffer = [0; NODE_SIZE]; + let mut buffer = [0; NODE_SIZE as usize]; // Number of leafs - let num_leafs = padded_piece_size.div_ceil(NODE_SIZE) as usize; + let num_leafs = piece_size.div_ceil(NODE_SIZE) as usize; // Elements iterator used by the MerkleTree. The elements returned by the // iterator represent leafs of the tree let elements_iterator = (0..num_leafs).map(|_| { - zero_padding_reader.read_exact(&mut buffer)?; + fr32_reader.read_exact(&mut buffer)?; let hash = Sha256Domain::try_from_bytes(&buffer)?; Ok(hash) }); @@ -117,7 +77,7 @@ pub fn calculate_piece_commitment( .map_err(|err| CommPError::TreeBuildError(err.to_string()))?; // Read and return the root of the tree - let mut commitment = [0; NODE_SIZE]; + let mut commitment = [0; NODE_SIZE as usize]; tree.root() .write_bytes(&mut commitment) .expect("destination buffer large enough"); @@ -125,13 +85,6 @@ pub fn calculate_piece_commitment( Ok(commitment) } -/// Generate Cid from the piece commitment -pub fn piece_commitment_cid(piece_commitment: [u8; 32]) -> Cid { - let hash = Multihash::wrap(SHA2_256_TRUNC254_PADDED, &piece_commitment) - .expect("piece commitment not more than 64 bytes"); - Cid::new_v1(FIL_COMMITMENT_UNSEALED, hash) -} - #[derive(Debug, Error)] pub enum CommPError { #[error("Piece is too small")] @@ -148,9 +101,9 @@ pub enum CommPError { mod tests { use std::io::Read; - use crate::commands::utils::commp::{ - calculate_piece_commitment, CommPError, ZeroPaddingReader, - }; + use primitives_shared::piece::PaddedPieceSize; + + use crate::commands::utils::commp::{calculate_piece_commitment, ZeroPaddingReader}; #[test] fn test_zero_padding_reader() { @@ -181,10 +134,14 @@ mod tests { fn test_calculate_piece_commitment() { use std::io::Cursor; - let data = vec![2u8; 200]; + let data_size: usize = 200; + let data = vec![2u8; data_size]; let cursor = Cursor::new(data.clone()); + let padded_piece_size = PaddedPieceSize::new(data_size.next_power_of_two() as u64).unwrap(); + let zero_padding_reader = ZeroPaddingReader::new(cursor, *padded_piece_size as usize); - let commitment = calculate_piece_commitment(cursor, data.len() as u64).unwrap(); + let commitment = + calculate_piece_commitment(zero_padding_reader, padded_piece_size).unwrap(); assert_eq!( commitment, [ @@ -192,12 +149,5 @@ mod tests { 148, 15, 250, 217, 3, 24, 152, 110, 93, 173, 117, 209, 251, 37, ] ); - - // Test with zero-length data - let empty_data = Vec::new(); - let empty_cursor = Cursor::new(empty_data); - - let empty_commitment = calculate_piece_commitment(empty_cursor, 0); - assert!(matches!(empty_commitment, Err(CommPError::PieceTooSmall))); } } diff --git a/cli/polka-storage-provider/src/commands/utils/mod.rs b/cli/polka-storage-provider/src/commands/utils/mod.rs index f10e8e568..940507020 100644 --- a/cli/polka-storage-provider/src/commands/utils/mod.rs +++ b/cli/polka-storage-provider/src/commands/utils/mod.rs @@ -5,9 +5,14 @@ use std::{fs::File, io::Write, path::PathBuf}; use mater::CarV2Reader; use polka_storage_proofs::porep; use primitives_proofs::RegisteredSealProof; +use primitives_shared::{ + commcid::piece_commitment_to_cid, + piece::PaddedPieceSize +}; +use std::io::BufReader; use crate::{ - commands::utils::commp::{calculate_piece_commitment, piece_commitment_cid, CommPError}, + commands::utils::commp::{calculate_piece_commitment, CommPError, ZeroPaddingReader}, CliError, }; @@ -47,13 +52,21 @@ impl UtilsCommand { let mut source_file = File::open(&input_path)?; let file_size = source_file.metadata()?.len(); + let buffered = BufReader::new(source_file); + let padded_piece_size = PaddedPieceSize::new(file_size.next_power_of_two() as u64) + .expect("is power of two"); + let mut zero_padding_reader = + ZeroPaddingReader::new(buffered, *padded_piece_size as usize); + // The calculate_piece_commitment blocks the thread. We could // use tokio::task::spawn_blocking to avoid this, but in this // case it doesn't matter because this is the only thing we are // working on. - let commitment = calculate_piece_commitment(&mut source_file, file_size) - .map_err(|err| UtilsCommandError::CommPError(err))?; - let cid = piece_commitment_cid(commitment); + let commitment = + calculate_piece_commitment(&mut zero_padding_reader, padded_piece_size) + .map_err(|err| UtilsCommandError::CommPError(err))?; + let cid = piece_commitment_to_cid(commitment) + .map_err(|err| UtilsCommandError::CidError(err.to_string()))?; println!("Piece commitment CID: {cid}"); } @@ -106,6 +119,8 @@ impl UtilsCommand { pub enum UtilsCommandError { #[error("the commp command failed because: {0}")] CommPError(#[from] CommPError), + #[error("CidError: {0}")] + CidError(String), #[error("failed to create a file '{0}' because: {1}")] FileCreateError(PathBuf, std::io::Error), #[error("failed to convert from rust-fil-proofs to polka-storage-proofs: {0}")] diff --git a/pallets/market/Cargo.toml b/pallets/market/Cargo.toml index f16a7bc15..495550fb4 100644 --- a/pallets/market/Cargo.toml +++ b/pallets/market/Cargo.toml @@ -22,7 +22,9 @@ hex = { workspace = true, default-features = false, features = ["alloc"] } log = { workspace = true } multihash-codetable = { workspace = true, features = ["blake2b"] } primitives-proofs = { workspace = true, default-features = false } +primitives-shared = { workspace = true } scale-info = { workspace = true, default-features = false, features = ["derive"] } +sha2 = { workspace = true } # frame deps frame-benchmarking = { workspace = true, default-features = false, optional = true } diff --git a/pallets/market/src/commd/mod.rs b/pallets/market/src/commd/mod.rs new file mode 100644 index 000000000..f5363a0b2 --- /dev/null +++ b/pallets/market/src/commd/mod.rs @@ -0,0 +1,290 @@ +extern crate alloc; +use alloc::vec::Vec; + +mod zero; + +use cid::Cid; +use primitives_proofs::SectorSize; +use primitives_shared::{commcid::Commitment, piece::PaddedPieceSize}; +use sha2::{Digest, Sha256}; + +use crate::commd::zero::zero_piece_commitment; + +// Ensure that the pieces are correct sizes +fn ensure_piece_sizes( + sector_size: SectorSize, + piece_infos: &[PieceInfo], +) -> Result<(), CommDError> { + let size_sum = piece_infos.iter().map(|piece| *piece.size).sum::(); + + if size_sum > sector_size.bytes() { + return Err(CommDError::PieceSizeTooLarge); + } + + Ok(()) +} + +/// Computes an unsealed sector CID (CommD) from its constituent piece CIDs (CommPs) and sizes. +pub fn compute_unsealed_sector_commitment( + sector_size: SectorSize, + piece_infos: &[PieceInfo], +) -> Result { + let padded_sector_size = PaddedPieceSize::new(sector_size.bytes()).unwrap(); + let unpadded_sector_size = padded_sector_size.unpadded(); + + // In case of no pieces, return the zero commitment for the whole sector. + if piece_infos.is_empty() { + return Ok(zero_piece_commitment(padded_sector_size)); + } + + // Check if pieces are correct sizes. + ensure_piece_sizes(sector_size, piece_infos)?; + + todo!(); + + // let commd = piece_reduction.commitment().unwrap(); + // Ok(commd) +} + +#[derive(Debug, Clone, Copy)] +pub struct PieceInfo { + /// Piece cid + pub cid: Cid, + /// Piece size + pub size: PaddedPieceSize, +} + +/// Stack used for piece reduction. +/// TODO: Temporary implementation copied from the filecoin +struct Stack(Vec); + +impl Stack { + /// Creates a new stack. + fn new() -> Self { + Stack(Vec::new()) + } + + /// Pushes a single element onto the stack. + fn shift(&mut self, el: PieceInfo) { + self.0.push(el) + } + + /// Look at the last element of the stack. + fn peek(&self) -> &PieceInfo { + &self.0[self.0.len() - 1] + } + + /// Look at the second to last element of the stack. + fn peek2(&self) -> &PieceInfo { + &self.0[self.0.len() - 2] + } + + /// Pop the last element of the stack. + fn pop(&mut self) -> PieceInfo { + self.0.pop().unwrap() + } + + fn reduce1(&mut self) -> bool { + if self.len() < 2 { + return false; + } + + if self.peek().size == self.peek2().size { + let right = self.pop(); + let left = self.pop(); + let joined = join_piece_infos(left, right).unwrap(); + self.shift(joined); + return true; + } + + false + } + + fn reduce(&mut self) { + while self.reduce1() {} + } + + fn shift_reduce(&mut self, piece: PieceInfo) { + self.shift(piece); + self.reduce() + } + + fn len(&self) -> usize { + self.0.len() + } +} + +/// Join two equally sized `PieceInfo`s together, by hashing them and adding +/// their sizes. +fn join_piece_infos(left: PieceInfo, right: PieceInfo) -> Result { + if left.size != right.size { + return Err(CommDError::InvalidPieceSize); + } + + todo!() + + // let commitment = piece_hash(&left.commitment, &right.commitment); + // let size = left.size + right.size; + + // Ok(PieceInfo { commitment, size }) +} + +/// Calculate Hash of two 32-byte arrays. +// pub fn piece_hash(a: &[u8], b: &[u8]) -> Sha256Domain { +// let mut buf = [0u8; NODE_SIZE * 2]; +// buf[..NODE_SIZE].copy_from_slice(a); +// buf[NODE_SIZE..].copy_from_slice(b); + +// // create a Sha256 object +// let mut hasher = Sha256::new(); +// hasher.update(buf); +// hasher.finalize() +// } + +#[derive(Debug)] +pub enum CommDError { + PieceSizeTooLarge, + InvalidPieceSize, +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use primitives_proofs::SectorSize; + use primitives_shared::commcid::data_commitment_to_cid; + + use super::*; + + #[test] + fn test_compute_comm_d_empty() { + let comm_d = compute_unsealed_sector_commitment(SectorSize::_2KiB, &[]) + .expect("failed to verify pieces, empty piece infos"); + assert_eq!( + comm_d, + [ + 252, 126, 146, 130, 150, 229, 22, 250, 173, 233, 134, 178, 143, 146, 212, 74, 79, + 36, 185, 53, 72, 82, 35, 55, 106, 121, 144, 39, 188, 24, 248, 51 + ] + ); + + let comm_d = zero_piece_commitment(PaddedPieceSize::new(2048).unwrap()); + assert_eq!( + comm_d, + [ + 252, 126, 146, 130, 150, 229, 22, 250, 173, 233, 134, 178, 143, 146, 212, 74, 79, + 36, 185, 53, 72, 82, 35, 55, 106, 121, 144, 39, 188, 24, 248, 51 + ] + ); + + let comm_d = zero_piece_commitment(PaddedPieceSize::new(128).unwrap()); + assert_eq!( + comm_d, + [ + 55, 49, 187, 153, 172, 104, 159, 102, 238, 245, 151, 62, 74, 148, 218, 24, 143, 77, + 220, 174, 88, 7, 36, 252, 111, 63, 214, 13, 253, 72, 131, 51 + ] + ); + } + + /// Reference: + #[test] + fn compute_unsealed_sector_cid() { + let pieces = vec![ + PieceInfo { + cid: Cid::from_str( + "baga6ea4seaqknzm22isnhsxt2s4dnw45kfywmhenngqq3nc7jvecakoca6ksyhy", + ) + .unwrap(), + size: PaddedPieceSize::new(256 << 20).unwrap(), + }, + PieceInfo { + cid: Cid::from_str( + "baga6ea4seaqnq6o5wuewdpviyoafno4rdpqnokz6ghvg2iyeyfbqxgcwdlj2egi", + ) + .unwrap(), + size: PaddedPieceSize::new(1024 << 20).unwrap(), + }, + PieceInfo { + cid: Cid::from_str( + "baga6ea4seaqpixk4ifbkzato3huzycj6ty6gllqwanhdpsvxikawyl5bg2h44mq", + ) + .unwrap(), + size: PaddedPieceSize::new(512 << 20).unwrap(), + }, + PieceInfo { + cid: Cid::from_str( + "baga6ea4seaqaxwe5dy6nt3ko5tngtmzvpqxqikw5mdwfjqgaxfwtzenc6bgzajq", + ) + .unwrap(), + size: PaddedPieceSize::new(512 << 20).unwrap(), + }, + PieceInfo { + cid: Cid::from_str( + "baga6ea4seaqpy33nbesa4d6ot2ygeuy43y4t7amc4izt52mlotqenwcmn2kyaai", + ) + .unwrap(), + size: PaddedPieceSize::new(1024 << 20).unwrap(), + }, + PieceInfo { + cid: Cid::from_str( + "baga6ea4seaqphvv4x2s2v7ykgc3ugs2kkltbdeg7icxstklkrgqvv72m2v3i2aa", + ) + .unwrap(), + size: PaddedPieceSize::new(256 << 20).unwrap(), + }, + PieceInfo { + cid: Cid::from_str( + "baga6ea4seaqf5u55znk6jwhdsrhe37emzhmehiyvjxpsww274f6fiy3h4yctady", + ) + .unwrap(), + size: PaddedPieceSize::new(512 << 20).unwrap(), + }, + PieceInfo { + cid: Cid::from_str( + "baga6ea4seaqa3qbabsbmvk5er6rhsjzt74beplzgulthamm22jue4zgqcuszofi", + ) + .unwrap(), + size: PaddedPieceSize::new(1024 << 20).unwrap(), + }, + PieceInfo { + cid: Cid::from_str( + "baga6ea4seaqiekvf623muj6jpxg6vsqaikyw3r4ob5u7363z7zcaixqvfqsc2ji", + ) + .unwrap(), + size: PaddedPieceSize::new(256 << 20).unwrap(), + }, + PieceInfo { + cid: Cid::from_str( + "baga6ea4seaqhsewv65z2d4m5o4vo65vl5o6z4bcegdvgnusvlt7rao44gro36pi", + ) + .unwrap(), + size: PaddedPieceSize::new(512 << 20).unwrap(), + }, + // The sector has to be filled entirely, before we can calculate the + // commitment, so we add two more empty pieces here. + PieceInfo { + cid: data_commitment_to_cid(zero_piece_commitment( + PaddedPieceSize::new(8 << 30).unwrap(), + )) + .unwrap(), + size: PaddedPieceSize::new(8 << 30).unwrap(), + }, + PieceInfo { + cid: data_commitment_to_cid(zero_piece_commitment( + PaddedPieceSize::new(16 << 30).unwrap(), + )) + .unwrap(), + size: PaddedPieceSize::new(16 << 30).unwrap(), + }, + ]; + + let comm_d = compute_unsealed_sector_commitment(SectorSize::_32GiB, &pieces).unwrap(); + let cid = data_commitment_to_cid(comm_d).unwrap(); + + assert_eq!( + cid.to_string(), + "baga6ea4seaqiw3gbmstmexb7sqwkc5r23o3i7zcyx5kr76pfobpykes3af62kca" + ); + } +} diff --git a/pallets/market/src/commd/zero.rs b/pallets/market/src/commd/zero.rs new file mode 100644 index 000000000..4be06917e --- /dev/null +++ b/pallets/market/src/commd/zero.rs @@ -0,0 +1,189 @@ +use primitives_shared::{commcid::Commitment, piece::PaddedPieceSize}; + +const LEVELS: usize = 37; +const SKIP: usize = 2; // can't generate for 32, 64b + +/// Returns a zero piece commitment based on the amount of space needed to pad. +pub fn zero_piece_commitment(sz: PaddedPieceSize) -> Commitment { + let level: usize = sz.trailing_zeros() as usize - SKIP - 5; // 2^5 = 32 + PIECE_COMMS[level] +} + +/// Zero piece commitments. This is statically defined to be able to pad remaining space with. +const PIECE_COMMS: [Commitment; LEVELS - SKIP] = [ + [ + 0x37, 0x31, 0xbb, 0x99, 0xac, 0x68, 0x9f, 0x66, 0xee, 0xf5, 0x97, 0x3e, 0x4a, 0x94, 0xda, + 0x18, 0x8f, 0x4d, 0xdc, 0xae, 0x58, 0x7, 0x24, 0xfc, 0x6f, 0x3f, 0xd6, 0xd, 0xfd, 0x48, + 0x83, 0x33, + ], + [ + 0x64, 0x2a, 0x60, 0x7e, 0xf8, 0x86, 0xb0, 0x4, 0xbf, 0x2c, 0x19, 0x78, 0x46, 0x3a, 0xe1, + 0xd4, 0x69, 0x3a, 0xc0, 0xf4, 0x10, 0xeb, 0x2d, 0x1b, 0x7a, 0x47, 0xfe, 0x20, 0x5e, 0x5e, + 0x75, 0xf, + ], + [ + 0x57, 0xa2, 0x38, 0x1a, 0x28, 0x65, 0x2b, 0xf4, 0x7f, 0x6b, 0xef, 0x7a, 0xca, 0x67, 0x9b, + 0xe4, 0xae, 0xde, 0x58, 0x71, 0xab, 0x5c, 0xf3, 0xeb, 0x2c, 0x8, 0x11, 0x44, 0x88, 0xcb, + 0x85, 0x26, + ], + [ + 0x1f, 0x7a, 0xc9, 0x59, 0x55, 0x10, 0xe0, 0x9e, 0xa4, 0x1c, 0x46, 0xb, 0x17, 0x64, 0x30, + 0xbb, 0x32, 0x2c, 0xd6, 0xfb, 0x41, 0x2e, 0xc5, 0x7c, 0xb1, 0x7d, 0x98, 0x9a, 0x43, 0x10, + 0x37, 0x2f, + ], + [ + 0xfc, 0x7e, 0x92, 0x82, 0x96, 0xe5, 0x16, 0xfa, 0xad, 0xe9, 0x86, 0xb2, 0x8f, 0x92, 0xd4, + 0x4a, 0x4f, 0x24, 0xb9, 0x35, 0x48, 0x52, 0x23, 0x37, 0x6a, 0x79, 0x90, 0x27, 0xbc, 0x18, + 0xf8, 0x33, + ], + [ + 0x8, 0xc4, 0x7b, 0x38, 0xee, 0x13, 0xbc, 0x43, 0xf4, 0x1b, 0x91, 0x5c, 0xe, 0xed, 0x99, + 0x11, 0xa2, 0x60, 0x86, 0xb3, 0xed, 0x62, 0x40, 0x1b, 0xf9, 0xd5, 0x8b, 0x8d, 0x19, 0xdf, + 0xf6, 0x24, + ], + [ + 0xb2, 0xe4, 0x7b, 0xfb, 0x11, 0xfa, 0xcd, 0x94, 0x1f, 0x62, 0xaf, 0x5c, 0x75, 0xf, 0x3e, + 0xa5, 0xcc, 0x4d, 0xf5, 0x17, 0xd5, 0xc4, 0xf1, 0x6d, 0xb2, 0xb4, 0xd7, 0x7b, 0xae, 0xc1, + 0xa3, 0x2f, + ], + [ + 0xf9, 0x22, 0x61, 0x60, 0xc8, 0xf9, 0x27, 0xbf, 0xdc, 0xc4, 0x18, 0xcd, 0xf2, 0x3, 0x49, + 0x31, 0x46, 0x0, 0x8e, 0xae, 0xfb, 0x7d, 0x2, 0x19, 0x4d, 0x5e, 0x54, 0x81, 0x89, 0x0, + 0x51, 0x8, + ], + [ + 0x2c, 0x1a, 0x96, 0x4b, 0xb9, 0xb, 0x59, 0xeb, 0xfe, 0xf, 0x6d, 0xa2, 0x9a, 0xd6, 0x5a, + 0xe3, 0xe4, 0x17, 0x72, 0x4a, 0x8f, 0x7c, 0x11, 0x74, 0x5a, 0x40, 0xca, 0xc1, 0xe5, 0xe7, + 0x40, 0x11, + ], + [ + 0xfe, 0xe3, 0x78, 0xce, 0xf1, 0x64, 0x4, 0xb1, 0x99, 0xed, 0xe0, 0xb1, 0x3e, 0x11, 0xb6, + 0x24, 0xff, 0x9d, 0x78, 0x4f, 0xbb, 0xed, 0x87, 0x8d, 0x83, 0x29, 0x7e, 0x79, 0x5e, 0x2, + 0x4f, 0x2, + ], + [ + 0x8e, 0x9e, 0x24, 0x3, 0xfa, 0x88, 0x4c, 0xf6, 0x23, 0x7f, 0x60, 0xdf, 0x25, 0xf8, 0x3e, + 0xe4, 0xd, 0xca, 0x9e, 0xd8, 0x79, 0xeb, 0x6f, 0x63, 0x52, 0xd1, 0x50, 0x84, 0xf5, 0xad, + 0xd, 0x3f, + ], + [ + 0x75, 0x2d, 0x96, 0x93, 0xfa, 0x16, 0x75, 0x24, 0x39, 0x54, 0x76, 0xe3, 0x17, 0xa9, 0x85, + 0x80, 0xf0, 0x9, 0x47, 0xaf, 0xb7, 0xa3, 0x5, 0x40, 0xd6, 0x25, 0xa9, 0x29, 0x1c, 0xc1, + 0x2a, 0x7, + ], + [ + 0x70, 0x22, 0xf6, 0xf, 0x7e, 0xf6, 0xad, 0xfa, 0x17, 0x11, 0x7a, 0x52, 0x61, 0x9e, 0x30, + 0xce, 0xa8, 0x2c, 0x68, 0x7, 0x5a, 0xdf, 0x1c, 0x66, 0x77, 0x86, 0xec, 0x50, 0x6e, 0xef, + 0x2d, 0x19, + ], + [ + 0xd9, 0x98, 0x87, 0xb9, 0x73, 0x57, 0x3a, 0x96, 0xe1, 0x13, 0x93, 0x64, 0x52, 0x36, 0xc1, + 0x7b, 0x1f, 0x4c, 0x70, 0x34, 0xd7, 0x23, 0xc7, 0xa9, 0x9f, 0x70, 0x9b, 0xb4, 0xda, 0x61, + 0x16, 0x2b, + ], + [ + 0xd0, 0xb5, 0x30, 0xdb, 0xb0, 0xb4, 0xf2, 0x5c, 0x5d, 0x2f, 0x2a, 0x28, 0xdf, 0xee, 0x80, + 0x8b, 0x53, 0x41, 0x2a, 0x2, 0x93, 0x1f, 0x18, 0xc4, 0x99, 0xf5, 0xa2, 0x54, 0x8, 0x6b, + 0x13, 0x26, + ], + [ + 0x84, 0xc0, 0x42, 0x1b, 0xa0, 0x68, 0x5a, 0x1, 0xbf, 0x79, 0x5a, 0x23, 0x44, 0x6, 0x4f, + 0xe4, 0x24, 0xbd, 0x52, 0xa9, 0xd2, 0x43, 0x77, 0xb3, 0x94, 0xff, 0x4c, 0x4b, 0x45, 0x68, + 0xe8, 0x11, + ], + [ + 0x65, 0xf2, 0x9e, 0x5d, 0x98, 0xd2, 0x46, 0xc3, 0x8b, 0x38, 0x8c, 0xfc, 0x6, 0xdb, 0x1f, + 0x6b, 0x2, 0x13, 0x3, 0xc5, 0xa2, 0x89, 0x0, 0xb, 0xdc, 0xe8, 0x32, 0xa9, 0xc3, 0xec, 0x42, + 0x1c, + ], + [ + 0xa2, 0x24, 0x75, 0x8, 0x28, 0x58, 0x50, 0x96, 0x5b, 0x7e, 0x33, 0x4b, 0x31, 0x27, 0xb0, + 0xc0, 0x42, 0xb1, 0xd0, 0x46, 0xdc, 0x54, 0x40, 0x21, 0x37, 0x62, 0x7c, 0xd8, 0x79, 0x9c, + 0xe1, 0x3a, + ], + [ + 0xda, 0xfd, 0xab, 0x6d, 0xa9, 0x36, 0x44, 0x53, 0xc2, 0x6d, 0x33, 0x72, 0x6b, 0x9f, 0xef, + 0xe3, 0x43, 0xbe, 0x8f, 0x81, 0x64, 0x9e, 0xc0, 0x9, 0xaa, 0xd3, 0xfa, 0xff, 0x50, 0x61, + 0x75, 0x8, + ], + [ + 0xd9, 0x41, 0xd5, 0xe0, 0xd6, 0x31, 0x4a, 0x99, 0x5c, 0x33, 0xff, 0xbd, 0x4f, 0xbe, 0x69, + 0x11, 0x8d, 0x73, 0xd4, 0xe5, 0xfd, 0x2c, 0xd3, 0x1f, 0xf, 0x7c, 0x86, 0xeb, 0xdd, 0x14, + 0xe7, 0x6, + ], + [ + 0x51, 0x4c, 0x43, 0x5c, 0x3d, 0x4, 0xd3, 0x49, 0xa5, 0x36, 0x5f, 0xbd, 0x59, 0xff, 0xc7, + 0x13, 0x62, 0x91, 0x11, 0x78, 0x59, 0x91, 0xc1, 0xa3, 0xc5, 0x3a, 0xf2, 0x20, 0x79, 0x74, + 0x1a, 0x2f, + ], + [ + 0xad, 0x6, 0x85, 0x39, 0x69, 0xd3, 0x7d, 0x34, 0xff, 0x8, 0xe0, 0x9f, 0x56, 0x93, 0xa, + 0x4a, 0xd1, 0x9a, 0x89, 0xde, 0xf6, 0xc, 0xbf, 0xee, 0x7e, 0x1d, 0x33, 0x81, 0xc1, 0xe7, + 0x1c, 0x37, + ], + [ + 0x39, 0x56, 0xe, 0x7b, 0x13, 0xa9, 0x3b, 0x7, 0xa2, 0x43, 0xfd, 0x27, 0x20, 0xff, 0xa7, + 0xcb, 0x3e, 0x1d, 0x2e, 0x50, 0x5a, 0xb3, 0x62, 0x9e, 0x79, 0xf4, 0x63, 0x13, 0x51, 0x2c, + 0xda, 0x6, + ], + [ + 0xcc, 0xc3, 0xc0, 0x12, 0xf5, 0xb0, 0x5e, 0x81, 0x1a, 0x2b, 0xbf, 0xdd, 0xf, 0x68, 0x33, + 0xb8, 0x42, 0x75, 0xb4, 0x7b, 0xf2, 0x29, 0xc0, 0x5, 0x2a, 0x82, 0x48, 0x4f, 0x3c, 0x1a, + 0x5b, 0x3d, + ], + [ + 0x7d, 0xf2, 0x9b, 0x69, 0x77, 0x31, 0x99, 0xe8, 0xf2, 0xb4, 0xb, 0x77, 0x91, 0x9d, 0x4, + 0x85, 0x9, 0xee, 0xd7, 0x68, 0xe2, 0xc7, 0x29, 0x7b, 0x1f, 0x14, 0x37, 0x3, 0x4f, 0xc3, + 0xc6, 0x2c, + ], + [ + 0x66, 0xce, 0x5, 0xa3, 0x66, 0x75, 0x52, 0xcf, 0x45, 0xc0, 0x2b, 0xcc, 0x4e, 0x83, 0x92, + 0x91, 0x9b, 0xde, 0xac, 0x35, 0xde, 0x2f, 0xf5, 0x62, 0x71, 0x84, 0x8e, 0x9f, 0x7b, 0x67, + 0x51, 0x7, + ], + [ + 0xd8, 0x61, 0x2, 0x18, 0x42, 0x5a, 0xb5, 0xe9, 0x5b, 0x1c, 0xa6, 0x23, 0x9d, 0x29, 0xa2, + 0xe4, 0x20, 0xd7, 0x6, 0xa9, 0x6f, 0x37, 0x3e, 0x2f, 0x9c, 0x9a, 0x91, 0xd7, 0x59, 0xd1, + 0x9b, 0x1, + ], + [ + 0x6d, 0x36, 0x4b, 0x1e, 0xf8, 0x46, 0x44, 0x1a, 0x5a, 0x4a, 0x68, 0x86, 0x23, 0x14, 0xac, + 0xc0, 0xa4, 0x6f, 0x1, 0x67, 0x17, 0xe5, 0x34, 0x43, 0xe8, 0x39, 0xee, 0xdf, 0x83, 0xc2, + 0x85, 0x3c, + ], + [ + 0x7, 0x7e, 0x5f, 0xde, 0x35, 0xc5, 0xa, 0x93, 0x3, 0xa5, 0x50, 0x9, 0xe3, 0x49, 0x8a, 0x4e, + 0xbe, 0xdf, 0xf3, 0x9c, 0x42, 0xb7, 0x10, 0xb7, 0x30, 0xd8, 0xec, 0x7a, 0xc7, 0xaf, 0xa6, + 0x3e, + ], + [ + 0xe6, 0x40, 0x5, 0xa6, 0xbf, 0xe3, 0x77, 0x79, 0x53, 0xb8, 0xad, 0x6e, 0xf9, 0x3f, 0xf, + 0xca, 0x10, 0x49, 0xb2, 0x4, 0x16, 0x54, 0xf2, 0xa4, 0x11, 0xf7, 0x70, 0x27, 0x99, 0xce, + 0xce, 0x2, + ], + [ + 0x25, 0x9d, 0x3d, 0x6b, 0x1f, 0x4d, 0x87, 0x6d, 0x11, 0x85, 0xe1, 0x12, 0x3a, 0xf6, 0xf5, + 0x50, 0x1a, 0xf0, 0xf6, 0x7c, 0xf1, 0x5b, 0x52, 0x16, 0x25, 0x5b, 0x7b, 0x17, 0x8d, 0x12, + 0x5, 0x1d, + ], + [ + 0x3f, 0x9a, 0x4d, 0x41, 0x1d, 0xa4, 0xef, 0x1b, 0x36, 0xf3, 0x5f, 0xf0, 0xa1, 0x95, 0xae, + 0x39, 0x2a, 0xb2, 0x3f, 0xee, 0x79, 0x67, 0xb7, 0xc4, 0x1b, 0x3, 0xd1, 0x61, 0x3f, 0xc2, + 0x92, 0x39, + ], + [ + 0xfe, 0x4e, 0xf3, 0x28, 0xc6, 0x1a, 0xa3, 0x9c, 0xfd, 0xb2, 0x48, 0x4e, 0xaa, 0x32, 0xa1, + 0x51, 0xb1, 0xfe, 0x3d, 0xfd, 0x1f, 0x96, 0xdd, 0x8c, 0x97, 0x11, 0xfd, 0x86, 0xd6, 0xc5, + 0x81, 0x13, + ], + [ + 0xf5, 0x5d, 0x68, 0x90, 0xe, 0x2d, 0x83, 0x81, 0xec, 0xcb, 0x81, 0x64, 0xcb, 0x99, 0x76, + 0xf2, 0x4b, 0x2d, 0xe0, 0xdd, 0x61, 0xa3, 0x1b, 0x97, 0xce, 0x6e, 0xb2, 0x38, 0x50, 0xd5, + 0xe8, 0x19, + ], + [ + 0xaa, 0xaa, 0x8c, 0x4c, 0xb4, 0xa, 0xac, 0xee, 0x1e, 0x2, 0xdc, 0x65, 0x42, 0x4b, 0x2a, + 0x6c, 0x8e, 0x99, 0xf8, 0x3, 0xb7, 0x2f, 0x79, 0x29, 0xc4, 0x10, 0x1d, 0x7f, 0xae, 0x6b, + 0xff, 0x32, + ], +]; diff --git a/pallets/market/src/lib.rs b/pallets/market/src/lib.rs index 249cdf89e..44093effd 100644 --- a/pallets/market/src/lib.rs +++ b/pallets/market/src/lib.rs @@ -7,6 +7,8 @@ #![cfg_attr(not(feature = "std"), no_std)] +mod commd; + pub use pallet::*; #[cfg(test)] @@ -42,10 +44,13 @@ pub mod pallet { SectorNumber, SectorSize, StorageProviderValidation, MAX_DEALS_FOR_ALL_SECTORS, MAX_DEALS_PER_SECTOR, MAX_SECTORS_PER_CALL, }; + use primitives_shared::{commcid::data_commitment_to_cid, piece::PaddedPieceSize}; use scale_info::TypeInfo; use sp_arithmetic::traits::BaseArithmetic; use sp_std::vec::Vec; + use crate::commd::compute_unsealed_sector_commitment; + pub const CID_CODEC: u64 = 0x55; pub const LOG_TARGET: &'static str = "runtime::market"; @@ -213,6 +218,9 @@ pub mod pallet { // It maybe doable using newtype pattern, however not sure how the UI on the frontend side would handle that anyways. // There is Encode/Decode implementation though, through the feature flag: `scale-codec`. pub piece_cid: BoundedVec>, + /// The value represents the size of the data piece after padding to the + /// nearest power of two. Padding ensures that all pieces can be + /// efficiently arranged in a binary tree structure for Merkle proofs. pub piece_size: u64, /// Storage Client's Account Id pub client: Address, @@ -244,8 +252,10 @@ pub mod pallet { pub state: DealState, } - impl - DealProposal + impl DealProposal + where + Balance: BaseArithmetic + Copy, + BlockNumber: BaseArithmetic + Copy, { fn duration(&self) -> BlockNumber { self.end_block - self.start_block @@ -947,15 +957,20 @@ pub mod pallet { /// fn compute_commd<'a>( - _proposals: impl IntoIterator>, - _sector_type: RegisteredSealProof, + proposals: impl IntoIterator>, + sector_type: RegisteredSealProof, ) -> Result { - // TODO(@th7nder,#92,21/06/2024): - // https://github.com/filecoin-project/rust-fil-proofs/blob/daec42b64ae6bf9a537545d5f116d57b9a29cc11/filecoin-proofs/src/pieces.rs#L85 - let cid = Cid::new_v1( - CID_CODEC, - Code::Blake2b256.digest(b"placeholder-to-be-done"), - ); + let pieces = proposals + .into_iter() + .map(|p| crate::commd::PieceInfo { + size: PaddedPieceSize::new(p.piece_size).unwrap(), + cid: p.cid().unwrap(), + }) + .collect::>(); + + let sector_size = sector_type.sector_size(); + let comm_d = compute_unsealed_sector_commitment(sector_size, &pieces).unwrap(); + let cid = data_commitment_to_cid(comm_d)?; Ok(cid) } diff --git a/primitives/proofs/src/types.rs b/primitives/proofs/src/types.rs index 4a0fcf20c..ccaf62bd9 100644 --- a/primitives/proofs/src/types.rs +++ b/primitives/proofs/src/types.rs @@ -20,6 +20,10 @@ pub type SectorNumber = u64; #[encode_as_type(crate_path = "::scale_encode")] pub enum SectorSize { _2KiB, + _8MiB, + _512MiB, + _32GiB, + _64GiB, } impl SectorSize { @@ -28,6 +32,10 @@ impl SectorSize { pub fn bytes(&self) -> u64 { match self { SectorSize::_2KiB => 2 << 10, + SectorSize::_8MiB => 8 << 20, + SectorSize::_512MiB => 512 << 20, + SectorSize::_32GiB => 32 << 30, + SectorSize::_64GiB => 2 * (32 << 30), } } } diff --git a/primitives/shared/Cargo.toml b/primitives/shared/Cargo.toml new file mode 100644 index 000000000..914dd89d4 --- /dev/null +++ b/primitives/shared/Cargo.toml @@ -0,0 +1,18 @@ +[package] +authors.workspace = true +edition.workspace = true +homepage.workspace = true +license-file.workspace = true +name = "primitives-shared" +repository.workspace = true +version = "0.1.0" + +[dependencies] +cid.workspace = true + +[dev-dependencies] + +[lints] +workspace = true + +[features] diff --git a/primitives/shared/src/commcid.rs b/primitives/shared/src/commcid.rs new file mode 100644 index 000000000..796153bbc --- /dev/null +++ b/primitives/shared/src/commcid.rs @@ -0,0 +1,88 @@ +use cid::{multihash::Multihash, Cid}; + +/// Filecoin piece or sector data commitment merkle node/root (CommP & CommD) +/// +/// https://github.com/multiformats/multicodec/blob/badcfe56bb7e0bbb06b60d57565186cd6be1f932/table.csv#L554 +pub const FIL_COMMITMENT_UNSEALED: u64 = 0xf101; + +/// Filecoin sector data commitment merkle node/root - sealed and replicated +/// (CommR) +/// +/// https://github.com/multiformats/multicodec/blob/badcfe56bb7e0bbb06b60d57565186cd6be1f932/table.csv#L555 +pub const FIL_COMMITMENT_SEALED: u64 = 0xf102; + +/// SHA2-256 with the two most significant bits from the last byte zeroed (as +/// via a mask with 0b00111111) - used for proving trees as in Filecoin. +/// +/// https://github.com/multiformats/multicodec/blob/badcfe56bb7e0bbb06b60d57565186cd6be1f932/table.csv#L153 +pub const SHA2_256_TRUNC254_PADDED: u64 = 0x1012; + +/// Poseidon using BLS12-381 and arity of 2 with Filecoin parameters +/// +/// https://github.com/multiformats/multicodec/blob/badcfe56bb7e0bbb06b60d57565186cd6be1f932/table.csv#L537 +pub const POSEIDON_BLS12_381_A1_FC1: u64 = 0xb401; + +/// Type representing a 32-byte commitment. +pub type Commitment = [u8; 32]; + +/// Converts a commitment to a CID. +pub fn commitment_to_cid( + multicodec: u64, + multihash: u64, + commitment: Commitment, +) -> Result { + validate_cid_segments(multicodec, multihash, &commitment)?; + + let hash = + Multihash::wrap(multihash, &commitment).map_err(|_| "failed to wrap commitment cid")?; + Ok(Cid::new_v1(multicodec, hash)) +} + +/// Destructure a CID to a commitment. +pub fn cid_to_commitment(cid: &Cid) -> Result<(u64, u64, Commitment), &'static str> { + validate_cid_segments(cid.codec(), cid.hash().code(), cid.hash().digest())?; + + let mut comm = Commitment::default(); + comm.copy_from_slice(cid.hash().digest()); + + Ok((cid.codec(), cid.hash().code(), comm)) +} + +/// Converts a piece commitment to a CID. +pub fn piece_commitment_to_cid(comm_p: Commitment) -> Result { + commitment_to_cid(FIL_COMMITMENT_UNSEALED, SHA2_256_TRUNC254_PADDED, comm_p) +} + +/// Converts a data commitment to a CID. +pub fn data_commitment_to_cid(comm_d: Commitment) -> Result { + commitment_to_cid(FIL_COMMITMENT_UNSEALED, SHA2_256_TRUNC254_PADDED, comm_d) +} + +/// Returns an error if the provided CID parts conflict with each other. +/// +/// Reference: +fn validate_cid_segments( + multicodec: u64, + multihash: u64, + commitment: &[u8], +) -> Result<(), &'static str> { + match multicodec { + FIL_COMMITMENT_UNSEALED => { + if multihash != SHA2_256_TRUNC254_PADDED { + return Err("Incorrect hash function for unsealed commitment"); + } + } + FIL_COMMITMENT_SEALED => { + if multihash != POSEIDON_BLS12_381_A1_FC1 { + return Err("Incorrect hash function for sealed commitment"); + } + } + _ => return Err("Invalid Codec, expected sealed or unsealed commitment codec"), + } + + if commitment.len() != 32 { + Err("commitments must be 32 bytes long") + } else { + Ok(()) + } +} diff --git a/primitives/shared/src/lib.rs b/primitives/shared/src/lib.rs new file mode 100644 index 000000000..11973f0c2 --- /dev/null +++ b/primitives/shared/src/lib.rs @@ -0,0 +1,8 @@ +#![no_std] + +pub mod commcid; +pub mod piece; + +/// Merkle tree node size in bytes. +/// TODO: Where should this be moved to? +pub const NODE_SIZE: u64 = 32; diff --git a/primitives/shared/src/piece.rs b/primitives/shared/src/piece.rs new file mode 100644 index 000000000..72816d807 --- /dev/null +++ b/primitives/shared/src/piece.rs @@ -0,0 +1,136 @@ +use core::ops::{Add, AddAssign, Deref}; + +use crate::NODE_SIZE; + +/// Size of a piece in bytes. Unpadded piece size should be power of two +/// multiple of 127. +#[derive(PartialEq, Debug, Eq, Clone, Copy)] +pub struct UnpaddedPieceSize(u64); + +impl UnpaddedPieceSize { + /// Initialize new unpadded piece size. Error is returned if the size is + /// invalid. + pub fn new(size: u64) -> Result { + if size < 127 { + return Err("minimum piece size is 127 bytes"); + } + + // is 127 * 2^n + if size >> size.trailing_zeros() != 127 { + return Err("unpadded piece size must be a power of 2 multiple of 127"); + } + + Ok(Self(size)) + } + + /// Converts unpadded piece size into padded piece size. + pub fn padded(self) -> PaddedPieceSize { + PaddedPieceSize(self.0 + (self.0 / 127)) + } +} + +impl Deref for UnpaddedPieceSize { + type Target = u64; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl Add for UnpaddedPieceSize { + type Output = Self; + + fn add(self, other: Self) -> Self::Output { + UnpaddedPieceSize(self.0 + other.0) + } +} + +/// Size of a piece in bytes with padding. The size should be power of two. +#[derive(PartialEq, Debug, Eq, Clone, Copy)] +pub struct PaddedPieceSize(u64); + +impl PaddedPieceSize { + /// Initialize new padded piece size. Error is returned if the size is + /// invalid. + pub fn new(size: u64) -> Result { + if size < 128 { + return Err("minimum piece size is 128 bytes"); + } + + if size.count_ones() != 1 { + return Err("padded piece size must be a power of 2"); + } + + if size % NODE_SIZE != 0 { + return Err("padded_piece_size is not multiple of NODE_SIZE"); + } + + Ok(Self(size)) + } + + /// Converts padded piece size into an unpadded piece size. + pub fn unpadded(self) -> UnpaddedPieceSize { + UnpaddedPieceSize(self.0 - (self.0 / 128)) + } +} + +impl Deref for PaddedPieceSize { + type Target = u64; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl Add for PaddedPieceSize { + type Output = Self; + + fn add(self, other: Self) -> Self::Output { + PaddedPieceSize(self.0 + other.0) + } +} + +impl AddAssign for PaddedPieceSize { + fn add_assign(&mut self, other: Self) { + self.0 += other.0; + } +} + +impl core::iter::Sum for PaddedPieceSize { + fn sum>(iter: I) -> Self { + iter.fold(PaddedPieceSize(0), |acc, x| acc + x) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn round_trip_piece_size() { + let p_piece = PaddedPieceSize::new(0b10000000).unwrap(); + let up_piece = p_piece.unpadded(); + assert_eq!(&up_piece, &UnpaddedPieceSize(127)); + assert_eq!(&p_piece, &up_piece.padded()); + } + #[test] + fn invalid_piece_checks() { + assert_eq!( + PaddedPieceSize::new(127), + Err("minimum piece size is 128 bytes") + ); + assert_eq!( + UnpaddedPieceSize::new(126), + Err("minimum piece size is 127 bytes") + ); + assert_eq!( + PaddedPieceSize::new(0b10000001), + Err("padded piece size must be a power of 2") + ); + assert_eq!( + UnpaddedPieceSize::new(0b1110111000), + Err("unpadded piece size must be a power of 2 multiple of 127") + ); + assert!(UnpaddedPieceSize::new(0b1111111000).is_ok()); + } +} From d760237e9d281c5a87b60d809c90fae8bea92d4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20=C4=8Cerni=C4=8D?= Date: Thu, 3 Oct 2024 11:47:50 +0200 Subject: [PATCH 02/24] change commd test --- .../src/commands/utils/commp.rs | 6 +- pallets/market/src/commd/mod.rs | 149 ++++++------------ pallets/market/src/lib.rs | 13 +- primitives/shared/src/lib.rs | 2 +- primitives/shared/src/piece.rs | 2 +- 5 files changed, 62 insertions(+), 110 deletions(-) diff --git a/cli/polka-storage-provider/src/commands/utils/commp.rs b/cli/polka-storage-provider/src/commands/utils/commp.rs index 337e4b8c0..32c07b9b5 100644 --- a/cli/polka-storage-provider/src/commands/utils/commp.rs +++ b/cli/polka-storage-provider/src/commands/utils/commp.rs @@ -62,9 +62,9 @@ pub fn calculate_piece_commitment( let mut fr32_reader = Fr32Reader::new(source); // Buffer used for reading data used for leafs. - let mut buffer = [0; NODE_SIZE as usize]; + let mut buffer = [0; NODE_SIZE]; // Number of leafs - let num_leafs = piece_size.div_ceil(NODE_SIZE) as usize; + let num_leafs = piece_size.div_ceil(NODE_SIZE as u64) as usize; // Elements iterator used by the MerkleTree. The elements returned by the // iterator represent leafs of the tree @@ -77,7 +77,7 @@ pub fn calculate_piece_commitment( .map_err(|err| CommPError::TreeBuildError(err.to_string()))?; // Read and return the root of the tree - let mut commitment = [0; NODE_SIZE as usize]; + let mut commitment = [0; NODE_SIZE]; tree.root() .write_bytes(&mut commitment) .expect("destination buffer large enough"); diff --git a/pallets/market/src/commd/mod.rs b/pallets/market/src/commd/mod.rs index f5363a0b2..e227ffb8c 100644 --- a/pallets/market/src/commd/mod.rs +++ b/pallets/market/src/commd/mod.rs @@ -5,7 +5,7 @@ mod zero; use cid::Cid; use primitives_proofs::SectorSize; -use primitives_shared::{commcid::Commitment, piece::PaddedPieceSize}; +use primitives_shared::{commcid::Commitment, piece::PaddedPieceSize, NODE_SIZE}; use sha2::{Digest, Sha256}; use crate::commd::zero::zero_piece_commitment; @@ -48,8 +48,8 @@ pub fn compute_unsealed_sector_commitment( #[derive(Debug, Clone, Copy)] pub struct PieceInfo { - /// Piece cid - pub cid: Cid, + /// Piece commitment + pub commitment: Commitment, /// Piece size pub size: PaddedPieceSize, } @@ -121,25 +121,25 @@ fn join_piece_infos(left: PieceInfo, right: PieceInfo) -> Result Sha256Domain { -// let mut buf = [0u8; NODE_SIZE * 2]; -// buf[..NODE_SIZE].copy_from_slice(a); -// buf[NODE_SIZE..].copy_from_slice(b); - -// // create a Sha256 object -// let mut hasher = Sha256::new(); -// hasher.update(buf); -// hasher.finalize() -// } +pub fn piece_hash(a: &[u8], b: &[u8]) -> Vec { + let mut buf = [0u8; NODE_SIZE * 2]; + buf[..NODE_SIZE].copy_from_slice(a); + buf[NODE_SIZE..].copy_from_slice(b); + + let mut hasher = Sha256::new(); + hasher.update(buf); + hasher.finalize().to_vec() +} #[derive(Debug)] pub enum CommDError { @@ -152,7 +152,7 @@ mod tests { use std::str::FromStr; use primitives_proofs::SectorSize; - use primitives_shared::commcid::data_commitment_to_cid; + use primitives_shared::commcid::{data_commitment_to_cid, cid_to_commitment}; use super::*; @@ -191,94 +191,39 @@ mod tests { #[test] fn compute_unsealed_sector_cid() { let pieces = vec![ - PieceInfo { - cid: Cid::from_str( - "baga6ea4seaqknzm22isnhsxt2s4dnw45kfywmhenngqq3nc7jvecakoca6ksyhy", - ) - .unwrap(), - size: PaddedPieceSize::new(256 << 20).unwrap(), - }, - PieceInfo { - cid: Cid::from_str( - "baga6ea4seaqnq6o5wuewdpviyoafno4rdpqnokz6ghvg2iyeyfbqxgcwdlj2egi", - ) - .unwrap(), - size: PaddedPieceSize::new(1024 << 20).unwrap(), - }, - PieceInfo { - cid: Cid::from_str( - "baga6ea4seaqpixk4ifbkzato3huzycj6ty6gllqwanhdpsvxikawyl5bg2h44mq", - ) - .unwrap(), - size: PaddedPieceSize::new(512 << 20).unwrap(), - }, - PieceInfo { - cid: Cid::from_str( - "baga6ea4seaqaxwe5dy6nt3ko5tngtmzvpqxqikw5mdwfjqgaxfwtzenc6bgzajq", - ) - .unwrap(), - size: PaddedPieceSize::new(512 << 20).unwrap(), - }, - PieceInfo { - cid: Cid::from_str( - "baga6ea4seaqpy33nbesa4d6ot2ygeuy43y4t7amc4izt52mlotqenwcmn2kyaai", - ) - .unwrap(), - size: PaddedPieceSize::new(1024 << 20).unwrap(), - }, - PieceInfo { - cid: Cid::from_str( - "baga6ea4seaqphvv4x2s2v7ykgc3ugs2kkltbdeg7icxstklkrgqvv72m2v3i2aa", - ) - .unwrap(), - size: PaddedPieceSize::new(256 << 20).unwrap(), - }, - PieceInfo { - cid: Cid::from_str( - "baga6ea4seaqf5u55znk6jwhdsrhe37emzhmehiyvjxpsww274f6fiy3h4yctady", - ) - .unwrap(), - size: PaddedPieceSize::new(512 << 20).unwrap(), - }, - PieceInfo { - cid: Cid::from_str( - "baga6ea4seaqa3qbabsbmvk5er6rhsjzt74beplzgulthamm22jue4zgqcuszofi", - ) - .unwrap(), - size: PaddedPieceSize::new(1024 << 20).unwrap(), - }, - PieceInfo { - cid: Cid::from_str( - "baga6ea4seaqiekvf623muj6jpxg6vsqaikyw3r4ob5u7363z7zcaixqvfqsc2ji", - ) - .unwrap(), - size: PaddedPieceSize::new(256 << 20).unwrap(), - }, - PieceInfo { - cid: Cid::from_str( - "baga6ea4seaqhsewv65z2d4m5o4vo65vl5o6z4bcegdvgnusvlt7rao44gro36pi", - ) - .unwrap(), - size: PaddedPieceSize::new(512 << 20).unwrap(), - }, + (Some("baga6ea4seaqknzm22isnhsxt2s4dnw45kfywmhenngqq3nc7jvecakoca6ksyhy"), 256 << 20), + (Some("baga6ea4seaqnq6o5wuewdpviyoafno4rdpqnokz6ghvg2iyeyfbqxgcwdlj2egi"), 1024 << 20), + (Some("baga6ea4seaqpixk4ifbkzato3huzycj6ty6gllqwanhdpsvxikawyl5bg2h44mq"), 512 << 20), + (Some("baga6ea4seaqaxwe5dy6nt3ko5tngtmzvpqxqikw5mdwfjqgaxfwtzenc6bgzajq"), 512 << 20), + (Some("baga6ea4seaqpy33nbesa4d6ot2ygeuy43y4t7amc4izt52mlotqenwcmn2kyaai"), 1024 << 20), + (Some("baga6ea4seaqphvv4x2s2v7ykgc3ugs2kkltbdeg7icxstklkrgqvv72m2v3i2aa"), 256 << 20), + (Some("baga6ea4seaqf5u55znk6jwhdsrhe37emzhmehiyvjxpsww274f6fiy3h4yctady"), 512 << 20), + (Some("baga6ea4seaqa3qbabsbmvk5er6rhsjzt74beplzgulthamm22jue4zgqcuszofi"), 1024 << 20), + (Some("baga6ea4seaqiekvf623muj6jpxg6vsqaikyw3r4ob5u7363z7zcaixqvfqsc2ji"), 256 << 20), + (Some("baga6ea4seaqhsewv65z2d4m5o4vo65vl5o6z4bcegdvgnusvlt7rao44gro36pi"), 512 << 20), // The sector has to be filled entirely, before we can calculate the // commitment, so we add two more empty pieces here. - PieceInfo { - cid: data_commitment_to_cid(zero_piece_commitment( - PaddedPieceSize::new(8 << 30).unwrap(), - )) - .unwrap(), - size: PaddedPieceSize::new(8 << 30).unwrap(), - }, - PieceInfo { - cid: data_commitment_to_cid(zero_piece_commitment( - PaddedPieceSize::new(16 << 30).unwrap(), - )) - .unwrap(), - size: PaddedPieceSize::new(16 << 30).unwrap(), - }, + (None, 8 << 30), + (None, 16 << 30) ]; + let pieces = pieces.into_iter().map(|(cid, size)| { + let size = PaddedPieceSize::new(size).unwrap(); + let commitment = match cid { + Some(cid) => { + let cid = Cid::from_str(cid).unwrap(); + let (_, _, commitment) = cid_to_commitment(&cid).unwrap(); + commitment + }, + None => zero_piece_commitment(size), + }; + + PieceInfo { + commitment, + size, + } + }).collect::>(); + let comm_d = compute_unsealed_sector_commitment(SectorSize::_32GiB, &pieces).unwrap(); let cid = data_commitment_to_cid(comm_d).unwrap(); diff --git a/pallets/market/src/lib.rs b/pallets/market/src/lib.rs index 44093effd..f8446d66f 100644 --- a/pallets/market/src/lib.rs +++ b/pallets/market/src/lib.rs @@ -39,6 +39,7 @@ pub mod pallet { }; use frame_system::{pallet_prelude::*, Config as SystemConfig, Pallet as System}; use multihash_codetable::{Code, MultihashDigest}; + use primitives_shared::commcid::cid_to_commitment; use primitives_proofs::{ ActiveDeal, ActiveSector, DealId, Market, RegisteredSealProof, SectorDeal, SectorId, SectorNumber, SectorSize, StorageProviderValidation, MAX_DEALS_FOR_ALL_SECTORS, @@ -962,9 +963,15 @@ pub mod pallet { ) -> Result { let pieces = proposals .into_iter() - .map(|p| crate::commd::PieceInfo { - size: PaddedPieceSize::new(p.piece_size).unwrap(), - cid: p.cid().unwrap(), + .map(|p| { + let cid = p.cid().unwrap(); + // TODO: Check the codec and hash type + let (_, _, commitment) = cid_to_commitment(&cid).unwrap(); + + crate::commd::PieceInfo { + size: PaddedPieceSize::new(p.piece_size).unwrap(), + commitment, + } }) .collect::>(); diff --git a/primitives/shared/src/lib.rs b/primitives/shared/src/lib.rs index 11973f0c2..1e7110331 100644 --- a/primitives/shared/src/lib.rs +++ b/primitives/shared/src/lib.rs @@ -5,4 +5,4 @@ pub mod piece; /// Merkle tree node size in bytes. /// TODO: Where should this be moved to? -pub const NODE_SIZE: u64 = 32; +pub const NODE_SIZE: usize = 32; diff --git a/primitives/shared/src/piece.rs b/primitives/shared/src/piece.rs index 72816d807..f5b8ab154 100644 --- a/primitives/shared/src/piece.rs +++ b/primitives/shared/src/piece.rs @@ -61,7 +61,7 @@ impl PaddedPieceSize { return Err("padded piece size must be a power of 2"); } - if size % NODE_SIZE != 0 { + if size % NODE_SIZE as u64 != 0 { return Err("padded_piece_size is not multiple of NODE_SIZE"); } From 459b138f38b021382c2831a87abcde5cd1c82f1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20=C4=8Cerni=C4=8D?= Date: Thu, 3 Oct 2024 13:01:55 +0200 Subject: [PATCH 03/24] refactor commitment handling to a struct --- .../src/commands/utils/commp.rs | 6 +- .../src/commands/utils/mod.rs | 6 +- pallets/market/src/{commd/mod.rs => commd.rs} | 32 ++--- pallets/market/src/lib.rs | 12 +- primitives/shared/src/commcid.rs | 88 -------------- primitives/shared/src/commitment/mod.rs | 109 ++++++++++++++++++ .../shared/src/commitment}/zero.rs | 6 +- primitives/shared/src/lib.rs | 2 +- 8 files changed, 141 insertions(+), 120 deletions(-) rename pallets/market/src/{commd/mod.rs => commd.rs} (91%) delete mode 100644 primitives/shared/src/commcid.rs create mode 100644 primitives/shared/src/commitment/mod.rs rename {pallets/market/src/commd => primitives/shared/src/commitment}/zero.rs (97%) diff --git a/cli/polka-storage-provider/src/commands/utils/commp.rs b/cli/polka-storage-provider/src/commands/utils/commp.rs index 32c07b9b5..adfe67885 100644 --- a/cli/polka-storage-provider/src/commands/utils/commp.rs +++ b/cli/polka-storage-provider/src/commands/utils/commp.rs @@ -5,7 +5,7 @@ use filecoin_hashers::{ Domain, }; use fr32::Fr32Reader; -use primitives_shared::{commcid::Commitment, piece::PaddedPieceSize, NODE_SIZE}; +use primitives_shared::{commitment::{Commitment, CommitmentKind}, piece::PaddedPieceSize, NODE_SIZE}; use storage_proofs_core::merkle::BinaryMerkleTree; use thiserror::Error; @@ -82,6 +82,8 @@ pub fn calculate_piece_commitment( .write_bytes(&mut commitment) .expect("destination buffer large enough"); + let commitment = Commitment::new(commitment, CommitmentKind::Piece); + Ok(commitment) } @@ -143,7 +145,7 @@ mod tests { let commitment = calculate_piece_commitment(zero_padding_reader, padded_piece_size).unwrap(); assert_eq!( - commitment, + commitment.raw(), [ 152, 58, 157, 235, 187, 58, 81, 61, 113, 252, 178, 149, 158, 13, 242, 24, 54, 98, 148, 15, 250, 217, 3, 24, 152, 110, 93, 173, 117, 209, 251, 37, diff --git a/cli/polka-storage-provider/src/commands/utils/mod.rs b/cli/polka-storage-provider/src/commands/utils/mod.rs index 940507020..b59178af2 100644 --- a/cli/polka-storage-provider/src/commands/utils/mod.rs +++ b/cli/polka-storage-provider/src/commands/utils/mod.rs @@ -6,7 +6,6 @@ use mater::CarV2Reader; use polka_storage_proofs::porep; use primitives_proofs::RegisteredSealProof; use primitives_shared::{ - commcid::piece_commitment_to_cid, piece::PaddedPieceSize }; use std::io::BufReader; @@ -49,7 +48,7 @@ impl UtilsCommand { car_v2_reader.is_car_file().await?; // Calculate the piece commitment. - let mut source_file = File::open(&input_path)?; + let source_file = File::open(&input_path)?; let file_size = source_file.metadata()?.len(); let buffered = BufReader::new(source_file); @@ -65,8 +64,7 @@ impl UtilsCommand { let commitment = calculate_piece_commitment(&mut zero_padding_reader, padded_piece_size) .map_err(|err| UtilsCommandError::CommPError(err))?; - let cid = piece_commitment_to_cid(commitment) - .map_err(|err| UtilsCommandError::CidError(err.to_string()))?; + let cid = commitment.cid(); println!("Piece commitment CID: {cid}"); } diff --git a/pallets/market/src/commd/mod.rs b/pallets/market/src/commd.rs similarity index 91% rename from pallets/market/src/commd/mod.rs rename to pallets/market/src/commd.rs index e227ffb8c..04877b65f 100644 --- a/pallets/market/src/commd/mod.rs +++ b/pallets/market/src/commd.rs @@ -1,14 +1,12 @@ extern crate alloc; use alloc::vec::Vec; -mod zero; - use cid::Cid; use primitives_proofs::SectorSize; -use primitives_shared::{commcid::Commitment, piece::PaddedPieceSize, NODE_SIZE}; +use primitives_shared::{commitment::Commitment, piece::PaddedPieceSize, NODE_SIZE}; use sha2::{Digest, Sha256}; - -use crate::commd::zero::zero_piece_commitment; +use primitives_shared::commitment::CommitmentKind; +use primitives_shared::commitment::zero_piece_commitment; // Ensure that the pieces are correct sizes fn ensure_piece_sizes( @@ -34,6 +32,7 @@ pub fn compute_unsealed_sector_commitment( // In case of no pieces, return the zero commitment for the whole sector. if piece_infos.is_empty() { + let comm = return Ok(zero_piece_commitment(padded_sector_size)); } @@ -121,13 +120,16 @@ fn join_piece_infos(left: PieceInfo, right: PieceInfo) -> Result { let cid = Cid::from_str(cid).unwrap(); - let (_, _, commitment) = cid_to_commitment(&cid).unwrap(); - commitment + Commitment::from_cid(&cid, CommitmentKind::Piece).unwrap() }, None => zero_piece_commitment(size), }; @@ -225,7 +225,7 @@ mod tests { }).collect::>(); let comm_d = compute_unsealed_sector_commitment(SectorSize::_32GiB, &pieces).unwrap(); - let cid = data_commitment_to_cid(comm_d).unwrap(); + let cid = comm_d.cid(); assert_eq!( cid.to_string(), diff --git a/pallets/market/src/lib.rs b/pallets/market/src/lib.rs index f8446d66f..fed50cd90 100644 --- a/pallets/market/src/lib.rs +++ b/pallets/market/src/lib.rs @@ -39,13 +39,15 @@ pub mod pallet { }; use frame_system::{pallet_prelude::*, Config as SystemConfig, Pallet as System}; use multihash_codetable::{Code, MultihashDigest}; - use primitives_shared::commcid::cid_to_commitment; + use primitives_shared::{ + piece::PaddedPieceSize, + commitment::{Commitment, CommitmentKind} + }; use primitives_proofs::{ ActiveDeal, ActiveSector, DealId, Market, RegisteredSealProof, SectorDeal, SectorId, SectorNumber, SectorSize, StorageProviderValidation, MAX_DEALS_FOR_ALL_SECTORS, MAX_DEALS_PER_SECTOR, MAX_SECTORS_PER_CALL, }; - use primitives_shared::{commcid::data_commitment_to_cid, piece::PaddedPieceSize}; use scale_info::TypeInfo; use sp_arithmetic::traits::BaseArithmetic; use sp_std::vec::Vec; @@ -965,8 +967,7 @@ pub mod pallet { .into_iter() .map(|p| { let cid = p.cid().unwrap(); - // TODO: Check the codec and hash type - let (_, _, commitment) = cid_to_commitment(&cid).unwrap(); + let commitment = Commitment::from_cid(&cid, CommitmentKind::Piece).unwrap(); crate::commd::PieceInfo { size: PaddedPieceSize::new(p.piece_size).unwrap(), @@ -977,9 +978,8 @@ pub mod pallet { let sector_size = sector_type.sector_size(); let comm_d = compute_unsealed_sector_commitment(sector_size, &pieces).unwrap(); - let cid = data_commitment_to_cid(comm_d)?; - Ok(cid) + Ok(comm_d.cid()) } /// diff --git a/primitives/shared/src/commcid.rs b/primitives/shared/src/commcid.rs deleted file mode 100644 index 796153bbc..000000000 --- a/primitives/shared/src/commcid.rs +++ /dev/null @@ -1,88 +0,0 @@ -use cid::{multihash::Multihash, Cid}; - -/// Filecoin piece or sector data commitment merkle node/root (CommP & CommD) -/// -/// https://github.com/multiformats/multicodec/blob/badcfe56bb7e0bbb06b60d57565186cd6be1f932/table.csv#L554 -pub const FIL_COMMITMENT_UNSEALED: u64 = 0xf101; - -/// Filecoin sector data commitment merkle node/root - sealed and replicated -/// (CommR) -/// -/// https://github.com/multiformats/multicodec/blob/badcfe56bb7e0bbb06b60d57565186cd6be1f932/table.csv#L555 -pub const FIL_COMMITMENT_SEALED: u64 = 0xf102; - -/// SHA2-256 with the two most significant bits from the last byte zeroed (as -/// via a mask with 0b00111111) - used for proving trees as in Filecoin. -/// -/// https://github.com/multiformats/multicodec/blob/badcfe56bb7e0bbb06b60d57565186cd6be1f932/table.csv#L153 -pub const SHA2_256_TRUNC254_PADDED: u64 = 0x1012; - -/// Poseidon using BLS12-381 and arity of 2 with Filecoin parameters -/// -/// https://github.com/multiformats/multicodec/blob/badcfe56bb7e0bbb06b60d57565186cd6be1f932/table.csv#L537 -pub const POSEIDON_BLS12_381_A1_FC1: u64 = 0xb401; - -/// Type representing a 32-byte commitment. -pub type Commitment = [u8; 32]; - -/// Converts a commitment to a CID. -pub fn commitment_to_cid( - multicodec: u64, - multihash: u64, - commitment: Commitment, -) -> Result { - validate_cid_segments(multicodec, multihash, &commitment)?; - - let hash = - Multihash::wrap(multihash, &commitment).map_err(|_| "failed to wrap commitment cid")?; - Ok(Cid::new_v1(multicodec, hash)) -} - -/// Destructure a CID to a commitment. -pub fn cid_to_commitment(cid: &Cid) -> Result<(u64, u64, Commitment), &'static str> { - validate_cid_segments(cid.codec(), cid.hash().code(), cid.hash().digest())?; - - let mut comm = Commitment::default(); - comm.copy_from_slice(cid.hash().digest()); - - Ok((cid.codec(), cid.hash().code(), comm)) -} - -/// Converts a piece commitment to a CID. -pub fn piece_commitment_to_cid(comm_p: Commitment) -> Result { - commitment_to_cid(FIL_COMMITMENT_UNSEALED, SHA2_256_TRUNC254_PADDED, comm_p) -} - -/// Converts a data commitment to a CID. -pub fn data_commitment_to_cid(comm_d: Commitment) -> Result { - commitment_to_cid(FIL_COMMITMENT_UNSEALED, SHA2_256_TRUNC254_PADDED, comm_d) -} - -/// Returns an error if the provided CID parts conflict with each other. -/// -/// Reference: -fn validate_cid_segments( - multicodec: u64, - multihash: u64, - commitment: &[u8], -) -> Result<(), &'static str> { - match multicodec { - FIL_COMMITMENT_UNSEALED => { - if multihash != SHA2_256_TRUNC254_PADDED { - return Err("Incorrect hash function for unsealed commitment"); - } - } - FIL_COMMITMENT_SEALED => { - if multihash != POSEIDON_BLS12_381_A1_FC1 { - return Err("Incorrect hash function for sealed commitment"); - } - } - _ => return Err("Invalid Codec, expected sealed or unsealed commitment codec"), - } - - if commitment.len() != 32 { - Err("commitments must be 32 bytes long") - } else { - Ok(()) - } -} diff --git a/primitives/shared/src/commitment/mod.rs b/primitives/shared/src/commitment/mod.rs new file mode 100644 index 000000000..3a3b074e1 --- /dev/null +++ b/primitives/shared/src/commitment/mod.rs @@ -0,0 +1,109 @@ +mod zero; + +use cid::{multihash::Multihash, Cid}; +use crate::piece::PaddedPieceSize; + +/// Filecoin piece or sector data commitment merkle node/root (CommP & CommD) +/// +/// https://github.com/multiformats/multicodec/blob/badcfe56bb7e0bbb06b60d57565186cd6be1f932/table.csv#L554 +pub const FIL_COMMITMENT_UNSEALED: u64 = 0xf101; + +/// Filecoin sector data commitment merkle node/root - sealed and replicated +/// (CommR) +/// +/// https://github.com/multiformats/multicodec/blob/badcfe56bb7e0bbb06b60d57565186cd6be1f932/table.csv#L555 +pub const FIL_COMMITMENT_SEALED: u64 = 0xf102; + +/// SHA2-256 with the two most significant bits from the last byte zeroed (as +/// via a mask with 0b00111111) - used for proving trees as in Filecoin. +/// +/// https://github.com/multiformats/multicodec/blob/badcfe56bb7e0bbb06b60d57565186cd6be1f932/table.csv#L153 +pub const SHA2_256_TRUNC254_PADDED: u64 = 0x1012; + +/// Poseidon using BLS12-381 and arity of 2 with Filecoin parameters +/// +/// https://github.com/multiformats/multicodec/blob/badcfe56bb7e0bbb06b60d57565186cd6be1f932/table.csv#L537 +pub const POSEIDON_BLS12_381_A1_FC1: u64 = 0xb401; + +#[derive(Debug, Clone, Copy)] +pub enum CommitmentKind { + // CommP - Piece commitment + Piece, + // CommD - Data commitment + Data +} + +impl CommitmentKind { + fn multicodec(&self) -> u64 { + match self { + CommitmentKind::Piece | CommitmentKind::Data => FIL_COMMITMENT_UNSEALED, + } + } + + fn multihash(&self) -> u64 { + match self { + CommitmentKind::Piece | CommitmentKind::Data => SHA2_256_TRUNC254_PADDED, + } + } +} + +#[derive(Debug, Clone, Copy)] +pub struct Commitment { + commitment: [u8; 32], + kind: CommitmentKind +} + +impl Commitment { + pub fn new(commitment: [u8; 32], kind: CommitmentKind) -> Self { + Self { + commitment, + kind + } + } + + pub fn from_cid(cid: &Cid, kind: CommitmentKind) -> Result { + let mut commitment = [0; 32]; + commitment.copy_from_slice(cid.hash().digest()); + + let multicodec = cid.codec(); + let multihash = cid.hash().code(); + + match kind { + CommitmentKind::Piece | CommitmentKind::Data => { + if multicodec != FIL_COMMITMENT_UNSEALED { + return Err("invalid multicodec for commitment"); + } + + if multihash != SHA2_256_TRUNC254_PADDED { + return Err("invalid multihash for commitment"); + } + } + } + + Ok(Self { + commitment, + kind + }) + } + + /// Returns the raw commitment bytes. + pub fn raw(&self) -> [u8; 32] { + self.commitment + } + + /// Converts the commitment to a CID. + pub fn cid(&self) -> Cid { + let multihash = self.kind.multihash(); + let multicodec = self.kind.multicodec(); + let hash = Multihash::wrap(multihash, &self.commitment).expect("correct commitment"); + Cid::new_v1(multicodec, hash) + } +} + +/// Returns a zero-piece commitment for a given piece size. +pub fn zero_piece_commitment(size: PaddedPieceSize) -> Commitment { + Commitment { + commitment: zero::zero_piece_commitment(size), + kind: CommitmentKind::Piece + } +} diff --git a/pallets/market/src/commd/zero.rs b/primitives/shared/src/commitment/zero.rs similarity index 97% rename from pallets/market/src/commd/zero.rs rename to primitives/shared/src/commitment/zero.rs index 4be06917e..973417101 100644 --- a/pallets/market/src/commd/zero.rs +++ b/primitives/shared/src/commitment/zero.rs @@ -1,16 +1,16 @@ -use primitives_shared::{commcid::Commitment, piece::PaddedPieceSize}; +use crate::piece::PaddedPieceSize; const LEVELS: usize = 37; const SKIP: usize = 2; // can't generate for 32, 64b /// Returns a zero piece commitment based on the amount of space needed to pad. -pub fn zero_piece_commitment(sz: PaddedPieceSize) -> Commitment { +pub fn zero_piece_commitment(sz: PaddedPieceSize) -> [u8; 32] { let level: usize = sz.trailing_zeros() as usize - SKIP - 5; // 2^5 = 32 PIECE_COMMS[level] } /// Zero piece commitments. This is statically defined to be able to pad remaining space with. -const PIECE_COMMS: [Commitment; LEVELS - SKIP] = [ +const PIECE_COMMS: [[u8; 32]; LEVELS - SKIP] = [ [ 0x37, 0x31, 0xbb, 0x99, 0xac, 0x68, 0x9f, 0x66, 0xee, 0xf5, 0x97, 0x3e, 0x4a, 0x94, 0xda, 0x18, 0x8f, 0x4d, 0xdc, 0xae, 0x58, 0x7, 0x24, 0xfc, 0x6f, 0x3f, 0xd6, 0xd, 0xfd, 0x48, diff --git a/primitives/shared/src/lib.rs b/primitives/shared/src/lib.rs index 1e7110331..75acb0126 100644 --- a/primitives/shared/src/lib.rs +++ b/primitives/shared/src/lib.rs @@ -1,6 +1,6 @@ #![no_std] -pub mod commcid; +pub mod commitment; pub mod piece; /// Merkle tree node size in bytes. From 4c2768f21fc8feafed35ca0dc3073d8ce53acb79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20=C4=8Cerni=C4=8D?= Date: Thu, 3 Oct 2024 13:35:00 +0200 Subject: [PATCH 04/24] remove ord impl --- pallets/market/src/commd.rs | 93 +++++++++++++++++++++++++++++++------ 1 file changed, 78 insertions(+), 15 deletions(-) diff --git a/pallets/market/src/commd.rs b/pallets/market/src/commd.rs index 04877b65f..14ca61d80 100644 --- a/pallets/market/src/commd.rs +++ b/pallets/market/src/commd.rs @@ -39,10 +39,45 @@ pub fn compute_unsealed_sector_commitment( // Check if pieces are correct sizes. ensure_piece_sizes(sector_size, piece_infos)?; - todo!(); + // Stack used for piece reduction + let mut stack = Stack::new(); + + // We first load the first piece into the stack. That first element is + // removed from the stack + let first = piece_infos + .first() + .expect("unreachable: !is_empty()") + .clone(); + // Push the first element on the stack + stack.shift(first); + + // Iterate all pieces + for piece_info in piece_infos.iter().skip(1) { + // Check if the last element of the stack is smaller than the current + // piece. If it is, add a padding piece to the stack. This is needed + // because when we reduce two pieces they have to be the same size. + while *stack.peek().size < *piece_info.size { + stack.shift_reduce(zero_padding(stack.peek().size)?)? + } + + // Push the actual piece onto the stack + stack.shift_reduce(piece_info.clone())?; + } + + // If we had odd number of pieces. add a 0 padded piece to the end of the stack + while stack.len() > 1 { + stack.shift_reduce(zero_padding(stack.peek().size)?)?; + } - // let commd = piece_reduction.commitment().unwrap(); - // Ok(commd) + // Stacks size must be 1. + if stack.len() != 1 { + return Err(CommDError::InvalidStackSize); + } + + // Last element on the stack is the actual comm_d of the piece + let commitment = stack.pop()?.commitment; + + Ok(commitment) } #[derive(Debug, Clone, Copy)] @@ -79,31 +114,32 @@ impl Stack { } /// Pop the last element of the stack. - fn pop(&mut self) -> PieceInfo { - self.0.pop().unwrap() + fn pop(&mut self) -> Result { + Ok(self.0.pop().unwrap()) } - fn reduce1(&mut self) -> bool { + fn reduce1(&mut self) -> Result { if self.len() < 2 { - return false; + return Ok(false); } if self.peek().size == self.peek2().size { - let right = self.pop(); - let left = self.pop(); - let joined = join_piece_infos(left, right).unwrap(); + let right = self.pop()?; + let left = self.pop()?; + let joined = join_piece_infos(left, right)?; self.shift(joined); - return true; + return Ok(true); } - false + Ok(false) } - fn reduce(&mut self) { - while self.reduce1() {} + fn reduce(&mut self) -> Result<(), CommDError> { + while self.reduce1()? {} + Ok(()) } - fn shift_reduce(&mut self, piece: PieceInfo) { + fn shift_reduce(&mut self, piece: PieceInfo) -> Result<(), CommDError> { self.shift(piece); self.reduce() } @@ -113,6 +149,32 @@ impl Stack { } } + +/// Create a padding `PieceInfo` of size `size`. +pub fn zero_padding(piece_size: PaddedPieceSize) -> Result { + let mut commitment = [0u8; 32]; + + // TODO: cache common piece hashes + let mut hashed_size = 64; + let h1 = piece_hash(&commitment, &commitment); + commitment.copy_from_slice(h1.as_ref()); + + while hashed_size < *piece_size { + let h = piece_hash(&commitment, &commitment); + commitment.copy_from_slice(h.as_ref()); + hashed_size *= 2; + } + + if hashed_size != *piece_size { + return Err(CommDError::InvalidPieceSize); + } + + Ok(PieceInfo { + commitment: Commitment::new(commitment, CommitmentKind::Piece), + size: piece_size + }) +} + /// Join two equally sized `PieceInfo`s together, by hashing them and adding /// their sizes. fn join_piece_infos(left: PieceInfo, right: PieceInfo) -> Result { @@ -147,6 +209,7 @@ pub fn piece_hash(a: &[u8], b: &[u8]) -> Vec { pub enum CommDError { PieceSizeTooLarge, InvalidPieceSize, + InvalidStackSize } #[cfg(test)] From 95358a2caf217a3f5418c3f03d21fa1912544ea9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20=C4=8Cerni=C4=8D?= Date: Thu, 3 Oct 2024 20:31:34 +0200 Subject: [PATCH 05/24] fix piece sizes --- Cargo.lock | 1 + .../src/commands/utils/commp.rs | 22 ++-- .../src/commands/utils/mod.rs | 17 ++- pallets/market/src/commd.rs | 100 ++++++++++++------ primitives/shared/Cargo.toml | 1 + primitives/shared/src/piece.rs | 14 ++- 6 files changed, 100 insertions(+), 55 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fd0c2036f..599c88922 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10865,6 +10865,7 @@ name = "primitives-shared" version = "0.1.0" dependencies = [ "cid 0.11.1", + "fr32", ] [[package]] diff --git a/cli/polka-storage-provider/src/commands/utils/commp.rs b/cli/polka-storage-provider/src/commands/utils/commp.rs index adfe67885..4dfc9c0f7 100644 --- a/cli/polka-storage-provider/src/commands/utils/commp.rs +++ b/cli/polka-storage-provider/src/commands/utils/commp.rs @@ -5,7 +5,11 @@ use filecoin_hashers::{ Domain, }; use fr32::Fr32Reader; -use primitives_shared::{commitment::{Commitment, CommitmentKind}, piece::PaddedPieceSize, NODE_SIZE}; +use primitives_shared::{ + commitment::{Commitment, CommitmentKind}, + piece::PaddedPieceSize, + NODE_SIZE, +}; use storage_proofs_core::merkle::BinaryMerkleTree; use thiserror::Error; @@ -14,11 +18,11 @@ pub struct ZeroPaddingReader { /// The inner reader to read from. inner: R, /// The number of bytes this 0-padding reader has left to produce. - remaining: usize, + remaining: u64, } impl ZeroPaddingReader { - pub fn new(inner: R, total_size: usize) -> Self { + pub fn new(inner: R, total_size: u64) -> Self { Self { inner, remaining: total_size, @@ -33,7 +37,7 @@ impl Read for ZeroPaddingReader { } // Number of bytes that the reader will produce in this execution - let to_read = buf.len().min(self.remaining); + let to_read = buf.len().min(self.remaining as usize); // Number of bytes that we read from the inner reader let read = self.inner.read(&mut buf[..to_read])?; @@ -44,7 +48,7 @@ impl Read for ZeroPaddingReader { } // Decrease the number of bytes this 0-padding reader has left to produce. - self.remaining -= to_read; + self.remaining -= to_read as u64; // Return the number of bytes that we wrote to the buffer. Ok(to_read) @@ -136,11 +140,11 @@ mod tests { fn test_calculate_piece_commitment() { use std::io::Cursor; - let data_size: usize = 200; - let data = vec![2u8; data_size]; + let data_size: u64 = 200; + let data = vec![2u8; data_size as usize]; let cursor = Cursor::new(data.clone()); - let padded_piece_size = PaddedPieceSize::new(data_size.next_power_of_two() as u64).unwrap(); - let zero_padding_reader = ZeroPaddingReader::new(cursor, *padded_piece_size as usize); + let padded_piece_size = PaddedPieceSize::from_arbitrary_size(data_size); + let zero_padding_reader = ZeroPaddingReader::new(cursor, *padded_piece_size); let commitment = calculate_piece_commitment(zero_padding_reader, padded_piece_size).unwrap(); diff --git a/cli/polka-storage-provider/src/commands/utils/mod.rs b/cli/polka-storage-provider/src/commands/utils/mod.rs index b59178af2..717ef8211 100644 --- a/cli/polka-storage-provider/src/commands/utils/mod.rs +++ b/cli/polka-storage-provider/src/commands/utils/mod.rs @@ -1,14 +1,15 @@ mod commp; -use std::{fs::File, io::Write, path::PathBuf}; +use std::{ + fs::File, + io::{BufReader, Write}, + path::PathBuf, +}; use mater::CarV2Reader; use polka_storage_proofs::porep; use primitives_proofs::RegisteredSealProof; -use primitives_shared::{ - piece::PaddedPieceSize -}; -use std::io::BufReader; +use primitives_shared::piece::PaddedPieceSize; use crate::{ commands::utils::commp::{calculate_piece_commitment, CommPError, ZeroPaddingReader}, @@ -52,10 +53,8 @@ impl UtilsCommand { let file_size = source_file.metadata()?.len(); let buffered = BufReader::new(source_file); - let padded_piece_size = PaddedPieceSize::new(file_size.next_power_of_two() as u64) - .expect("is power of two"); - let mut zero_padding_reader = - ZeroPaddingReader::new(buffered, *padded_piece_size as usize); + let padded_piece_size = PaddedPieceSize::from_arbitrary_size(file_size as u64); + let mut zero_padding_reader = ZeroPaddingReader::new(buffered, *padded_piece_size); // The calculate_piece_commitment blocks the thread. We could // use tokio::task::spawn_blocking to avoid this, but in this diff --git a/pallets/market/src/commd.rs b/pallets/market/src/commd.rs index 14ca61d80..1f339ffa4 100644 --- a/pallets/market/src/commd.rs +++ b/pallets/market/src/commd.rs @@ -3,10 +3,12 @@ use alloc::vec::Vec; use cid::Cid; use primitives_proofs::SectorSize; -use primitives_shared::{commitment::Commitment, piece::PaddedPieceSize, NODE_SIZE}; +use primitives_shared::{ + commitment::{zero_piece_commitment, Commitment, CommitmentKind}, + piece::PaddedPieceSize, + NODE_SIZE, +}; use sha2::{Digest, Sha256}; -use primitives_shared::commitment::CommitmentKind; -use primitives_shared::commitment::zero_piece_commitment; // Ensure that the pieces are correct sizes fn ensure_piece_sizes( @@ -32,8 +34,7 @@ pub fn compute_unsealed_sector_commitment( // In case of no pieces, return the zero commitment for the whole sector. if piece_infos.is_empty() { - let comm = - return Ok(zero_piece_commitment(padded_sector_size)); + let comm = return Ok(zero_piece_commitment(padded_sector_size)); } // Check if pieces are correct sizes. @@ -149,7 +150,6 @@ impl Stack { } } - /// Create a padding `PieceInfo` of size `size`. pub fn zero_padding(piece_size: PaddedPieceSize) -> Result { let mut commitment = [0u8; 32]; @@ -171,7 +171,7 @@ pub fn zero_padding(piece_size: PaddedPieceSize) -> Result Result Vec { pub enum CommDError { PieceSizeTooLarge, InvalidPieceSize, - InvalidStackSize + InvalidStackSize, } #[cfg(test)] @@ -255,37 +255,67 @@ mod tests { #[test] fn compute_unsealed_sector_cid() { let pieces = vec![ - (Some("baga6ea4seaqknzm22isnhsxt2s4dnw45kfywmhenngqq3nc7jvecakoca6ksyhy"), 256 << 20), - (Some("baga6ea4seaqnq6o5wuewdpviyoafno4rdpqnokz6ghvg2iyeyfbqxgcwdlj2egi"), 1024 << 20), - (Some("baga6ea4seaqpixk4ifbkzato3huzycj6ty6gllqwanhdpsvxikawyl5bg2h44mq"), 512 << 20), - (Some("baga6ea4seaqaxwe5dy6nt3ko5tngtmzvpqxqikw5mdwfjqgaxfwtzenc6bgzajq"), 512 << 20), - (Some("baga6ea4seaqpy33nbesa4d6ot2ygeuy43y4t7amc4izt52mlotqenwcmn2kyaai"), 1024 << 20), - (Some("baga6ea4seaqphvv4x2s2v7ykgc3ugs2kkltbdeg7icxstklkrgqvv72m2v3i2aa"), 256 << 20), - (Some("baga6ea4seaqf5u55znk6jwhdsrhe37emzhmehiyvjxpsww274f6fiy3h4yctady"), 512 << 20), - (Some("baga6ea4seaqa3qbabsbmvk5er6rhsjzt74beplzgulthamm22jue4zgqcuszofi"), 1024 << 20), - (Some("baga6ea4seaqiekvf623muj6jpxg6vsqaikyw3r4ob5u7363z7zcaixqvfqsc2ji"), 256 << 20), - (Some("baga6ea4seaqhsewv65z2d4m5o4vo65vl5o6z4bcegdvgnusvlt7rao44gro36pi"), 512 << 20), + ( + Some("baga6ea4seaqknzm22isnhsxt2s4dnw45kfywmhenngqq3nc7jvecakoca6ksyhy"), + 256 << 20, + ), + ( + Some("baga6ea4seaqnq6o5wuewdpviyoafno4rdpqnokz6ghvg2iyeyfbqxgcwdlj2egi"), + 1024 << 20, + ), + ( + Some("baga6ea4seaqpixk4ifbkzato3huzycj6ty6gllqwanhdpsvxikawyl5bg2h44mq"), + 512 << 20, + ), + ( + Some("baga6ea4seaqaxwe5dy6nt3ko5tngtmzvpqxqikw5mdwfjqgaxfwtzenc6bgzajq"), + 512 << 20, + ), + ( + Some("baga6ea4seaqpy33nbesa4d6ot2ygeuy43y4t7amc4izt52mlotqenwcmn2kyaai"), + 1024 << 20, + ), + ( + Some("baga6ea4seaqphvv4x2s2v7ykgc3ugs2kkltbdeg7icxstklkrgqvv72m2v3i2aa"), + 256 << 20, + ), + ( + Some("baga6ea4seaqf5u55znk6jwhdsrhe37emzhmehiyvjxpsww274f6fiy3h4yctady"), + 512 << 20, + ), + ( + Some("baga6ea4seaqa3qbabsbmvk5er6rhsjzt74beplzgulthamm22jue4zgqcuszofi"), + 1024 << 20, + ), + ( + Some("baga6ea4seaqiekvf623muj6jpxg6vsqaikyw3r4ob5u7363z7zcaixqvfqsc2ji"), + 256 << 20, + ), + ( + Some("baga6ea4seaqhsewv65z2d4m5o4vo65vl5o6z4bcegdvgnusvlt7rao44gro36pi"), + 512 << 20, + ), // The sector has to be filled entirely, before we can calculate the // commitment, so we add two more empty pieces here. (None, 8 << 30), - (None, 16 << 30) + (None, 16 << 30), ]; - let pieces = pieces.into_iter().map(|(cid, size)| { - let size = PaddedPieceSize::new(size).unwrap(); - let commitment = match cid { - Some(cid) => { - let cid = Cid::from_str(cid).unwrap(); - Commitment::from_cid(&cid, CommitmentKind::Piece).unwrap() - }, - None => zero_piece_commitment(size), - }; - - PieceInfo { - commitment, - size, - } - }).collect::>(); + let pieces = pieces + .into_iter() + .map(|(cid, size)| { + let size = PaddedPieceSize::new(size).unwrap(); + let commitment = match cid { + Some(cid) => { + let cid = Cid::from_str(cid).unwrap(); + Commitment::from_cid(&cid, CommitmentKind::Piece).unwrap() + } + None => zero_piece_commitment(size), + }; + + PieceInfo { commitment, size } + }) + .collect::>(); let comm_d = compute_unsealed_sector_commitment(SectorSize::_32GiB, &pieces).unwrap(); let cid = comm_d.cid(); diff --git a/primitives/shared/Cargo.toml b/primitives/shared/Cargo.toml index 914dd89d4..53331fd71 100644 --- a/primitives/shared/Cargo.toml +++ b/primitives/shared/Cargo.toml @@ -9,6 +9,7 @@ version = "0.1.0" [dependencies] cid.workspace = true +fr32.workspace = true [dev-dependencies] diff --git a/primitives/shared/src/piece.rs b/primitives/shared/src/piece.rs index f5b8ab154..127d1a40c 100644 --- a/primitives/shared/src/piece.rs +++ b/primitives/shared/src/piece.rs @@ -1,5 +1,7 @@ use core::ops::{Add, AddAssign, Deref}; +use fr32::{to_padded_bytes, to_unpadded_bytes}; + use crate::NODE_SIZE; /// Size of a piece in bytes. Unpadded piece size should be power of two @@ -25,7 +27,7 @@ impl UnpaddedPieceSize { /// Converts unpadded piece size into padded piece size. pub fn padded(self) -> PaddedPieceSize { - PaddedPieceSize(self.0 + (self.0 / 127)) + PaddedPieceSize(to_padded_bytes(self.0 as usize) as u64) } } @@ -70,7 +72,15 @@ impl PaddedPieceSize { /// Converts padded piece size into an unpadded piece size. pub fn unpadded(self) -> UnpaddedPieceSize { - UnpaddedPieceSize(self.0 - (self.0 / 128)) + UnpaddedPieceSize(to_unpadded_bytes(self.0)) + } + + /// The function accepts arbitrary size and transforms it to the + /// PaddedPieceSize. We first pad the size. After that we take the first + /// power of 2 number. + pub fn from_arbitrary_size(size: u64) -> Self { + let padded = to_padded_bytes(size as usize).next_power_of_two(); + Self::new(padded as u64).expect("the padded piece size is correct") } } From 16154114e67f1ece13a59bcdb3c1873eaef24459 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20=C4=8Cerni=C4=8D?= Date: Thu, 3 Oct 2024 20:42:52 +0200 Subject: [PATCH 06/24] return size when calculating commp --- cli/polka-storage-provider/src/commands/utils/mod.rs | 1 + primitives/shared/src/piece.rs | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/cli/polka-storage-provider/src/commands/utils/mod.rs b/cli/polka-storage-provider/src/commands/utils/mod.rs index 717ef8211..e44d4e399 100644 --- a/cli/polka-storage-provider/src/commands/utils/mod.rs +++ b/cli/polka-storage-provider/src/commands/utils/mod.rs @@ -66,6 +66,7 @@ impl UtilsCommand { let cid = commitment.cid(); println!("Piece commitment CID: {cid}"); + println!("Padded size: {padded_piece_size}"); } UtilsCommand::GeneratePoRepParams { seal_proof, diff --git a/primitives/shared/src/piece.rs b/primitives/shared/src/piece.rs index 127d1a40c..f5e9d57eb 100644 --- a/primitives/shared/src/piece.rs +++ b/primitives/shared/src/piece.rs @@ -31,6 +31,12 @@ impl UnpaddedPieceSize { } } +impl core::fmt::Display for UnpaddedPieceSize { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "{}", self.0) + } +} + impl Deref for UnpaddedPieceSize { type Target = u64; @@ -84,6 +90,12 @@ impl PaddedPieceSize { } } +impl core::fmt::Display for PaddedPieceSize { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "{}", self.0) + } +} + impl Deref for PaddedPieceSize { type Target = u64; From c27f36da01d50ed59a0b943cb17fd1789bd5baac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20=C4=8Cerni=C4=8D?= Date: Fri, 4 Oct 2024 11:41:27 +0200 Subject: [PATCH 07/24] fix build --- Cargo.lock | 1 - pallets/market/src/commd.rs | 14 +++++--------- pallets/market/src/lib.rs | 9 ++++----- primitives/shared/Cargo.toml | 1 - primitives/shared/src/piece.rs | 13 +++++++------ 5 files changed, 16 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 599c88922..fd0c2036f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10865,7 +10865,6 @@ name = "primitives-shared" version = "0.1.0" dependencies = [ "cid 0.11.1", - "fr32", ] [[package]] diff --git a/pallets/market/src/commd.rs b/pallets/market/src/commd.rs index 1f339ffa4..623fda067 100644 --- a/pallets/market/src/commd.rs +++ b/pallets/market/src/commd.rs @@ -1,7 +1,6 @@ extern crate alloc; use alloc::vec::Vec; -use cid::Cid; use primitives_proofs::SectorSize; use primitives_shared::{ commitment::{zero_piece_commitment, Commitment, CommitmentKind}, @@ -30,11 +29,10 @@ pub fn compute_unsealed_sector_commitment( piece_infos: &[PieceInfo], ) -> Result { let padded_sector_size = PaddedPieceSize::new(sector_size.bytes()).unwrap(); - let unpadded_sector_size = padded_sector_size.unpadded(); // In case of no pieces, return the zero commitment for the whole sector. if piece_infos.is_empty() { - let comm = return Ok(zero_piece_commitment(padded_sector_size)); + return Ok(zero_piece_commitment(padded_sector_size)); } // Check if pieces are correct sizes. @@ -45,12 +43,9 @@ pub fn compute_unsealed_sector_commitment( // We first load the first piece into the stack. That first element is // removed from the stack - let first = piece_infos - .first() - .expect("unreachable: !is_empty()") - .clone(); + let first = piece_infos.first().expect("unreachable: !is_empty()"); // Push the first element on the stack - stack.shift(first); + stack.shift(*first); // Iterate all pieces for piece_info in piece_infos.iter().skip(1) { @@ -62,7 +57,7 @@ pub fn compute_unsealed_sector_commitment( } // Push the actual piece onto the stack - stack.shift_reduce(piece_info.clone())?; + stack.shift_reduce(*piece_info)?; } // If we had odd number of pieces. add a 0 padded piece to the end of the stack @@ -216,6 +211,7 @@ pub enum CommDError { mod tests { use std::str::FromStr; + use cid::Cid; use primitives_proofs::SectorSize; use super::*; diff --git a/pallets/market/src/lib.rs b/pallets/market/src/lib.rs index fed50cd90..cf9a22054 100644 --- a/pallets/market/src/lib.rs +++ b/pallets/market/src/lib.rs @@ -38,16 +38,15 @@ pub mod pallet { PalletId, }; use frame_system::{pallet_prelude::*, Config as SystemConfig, Pallet as System}; - use multihash_codetable::{Code, MultihashDigest}; - use primitives_shared::{ - piece::PaddedPieceSize, - commitment::{Commitment, CommitmentKind} - }; use primitives_proofs::{ ActiveDeal, ActiveSector, DealId, Market, RegisteredSealProof, SectorDeal, SectorId, SectorNumber, SectorSize, StorageProviderValidation, MAX_DEALS_FOR_ALL_SECTORS, MAX_DEALS_PER_SECTOR, MAX_SECTORS_PER_CALL, }; + use primitives_shared::{ + commitment::{Commitment, CommitmentKind}, + piece::PaddedPieceSize, + }; use scale_info::TypeInfo; use sp_arithmetic::traits::BaseArithmetic; use sp_std::vec::Vec; diff --git a/primitives/shared/Cargo.toml b/primitives/shared/Cargo.toml index 53331fd71..914dd89d4 100644 --- a/primitives/shared/Cargo.toml +++ b/primitives/shared/Cargo.toml @@ -9,7 +9,6 @@ version = "0.1.0" [dependencies] cid.workspace = true -fr32.workspace = true [dev-dependencies] diff --git a/primitives/shared/src/piece.rs b/primitives/shared/src/piece.rs index f5e9d57eb..5deae2eb9 100644 --- a/primitives/shared/src/piece.rs +++ b/primitives/shared/src/piece.rs @@ -1,7 +1,5 @@ use core::ops::{Add, AddAssign, Deref}; -use fr32::{to_padded_bytes, to_unpadded_bytes}; - use crate::NODE_SIZE; /// Size of a piece in bytes. Unpadded piece size should be power of two @@ -27,7 +25,8 @@ impl UnpaddedPieceSize { /// Converts unpadded piece size into padded piece size. pub fn padded(self) -> PaddedPieceSize { - PaddedPieceSize(to_padded_bytes(self.0 as usize) as u64) + let padded_bytes = self.0 + (self.0 / 127); + PaddedPieceSize(padded_bytes) } } @@ -78,15 +77,17 @@ impl PaddedPieceSize { /// Converts padded piece size into an unpadded piece size. pub fn unpadded(self) -> UnpaddedPieceSize { - UnpaddedPieceSize(to_unpadded_bytes(self.0)) + let unpadded_bytes = self.0 - (self.0 / 128); + UnpaddedPieceSize(unpadded_bytes) } /// The function accepts arbitrary size and transforms it to the /// PaddedPieceSize. We first pad the size. After that we take the first /// power of 2 number. pub fn from_arbitrary_size(size: u64) -> Self { - let padded = to_padded_bytes(size as usize).next_power_of_two(); - Self::new(padded as u64).expect("the padded piece size is correct") + let padded_bytes = size + (size / 127); + let padded_bytes = padded_bytes.next_power_of_two(); + Self::new(padded_bytes as u64).expect("the padded piece size is correct") } } From 0f4db1f9cc1af3e9f79d134ae6e2d7b0f2eb1168 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20=C4=8Cerni=C4=8D?= Date: Fri, 4 Oct 2024 11:53:36 +0200 Subject: [PATCH 08/24] add replica commitment kind --- primitives/shared/src/commitment/mod.rs | 26 ++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/primitives/shared/src/commitment/mod.rs b/primitives/shared/src/commitment/mod.rs index 3a3b074e1..370ff3896 100644 --- a/primitives/shared/src/commitment/mod.rs +++ b/primitives/shared/src/commitment/mod.rs @@ -1,6 +1,7 @@ mod zero; use cid::{multihash::Multihash, Cid}; + use crate::piece::PaddedPieceSize; /// Filecoin piece or sector data commitment merkle node/root (CommP & CommD) @@ -30,19 +31,23 @@ pub enum CommitmentKind { // CommP - Piece commitment Piece, // CommD - Data commitment - Data + Data, + // CommR - Replica commitment + Replica, } impl CommitmentKind { fn multicodec(&self) -> u64 { match self { CommitmentKind::Piece | CommitmentKind::Data => FIL_COMMITMENT_UNSEALED, + CommitmentKind::Replica => FIL_COMMITMENT_SEALED, } } fn multihash(&self) -> u64 { match self { CommitmentKind::Piece | CommitmentKind::Data => SHA2_256_TRUNC254_PADDED, + CommitmentKind::Replica => POSEIDON_BLS12_381_A1_FC1, } } } @@ -50,15 +55,12 @@ impl CommitmentKind { #[derive(Debug, Clone, Copy)] pub struct Commitment { commitment: [u8; 32], - kind: CommitmentKind + kind: CommitmentKind, } impl Commitment { pub fn new(commitment: [u8; 32], kind: CommitmentKind) -> Self { - Self { - commitment, - kind - } + Self { commitment, kind } } pub fn from_cid(cid: &Cid, kind: CommitmentKind) -> Result { @@ -78,12 +80,14 @@ impl Commitment { return Err("invalid multihash for commitment"); } } + CommitmentKind::Replica => { + if multicodec != FIL_COMMITMENT_SEALED { + return Err("invalid multicodec for commitment"); + } + } } - Ok(Self { - commitment, - kind - }) + Ok(Self { commitment, kind }) } /// Returns the raw commitment bytes. @@ -104,6 +108,6 @@ impl Commitment { pub fn zero_piece_commitment(size: PaddedPieceSize) -> Commitment { Commitment { commitment: zero::zero_piece_commitment(size), - kind: CommitmentKind::Piece + kind: CommitmentKind::Piece, } } From da3551a6dc803db890f551314be0d2eb8ac4ba3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20=C4=8Cerni=C4=8D?= Date: Fri, 4 Oct 2024 14:32:47 +0200 Subject: [PATCH 09/24] add tests for commitment --- Cargo.lock | 1 + primitives/shared/Cargo.toml | 1 + primitives/shared/src/commitment/mod.rs | 122 +++++++++++++++++++++++- 3 files changed, 122 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fd0c2036f..ca4612b0c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10865,6 +10865,7 @@ name = "primitives-shared" version = "0.1.0" dependencies = [ "cid 0.11.1", + "rand", ] [[package]] diff --git a/primitives/shared/Cargo.toml b/primitives/shared/Cargo.toml index 914dd89d4..16ad9b287 100644 --- a/primitives/shared/Cargo.toml +++ b/primitives/shared/Cargo.toml @@ -11,6 +11,7 @@ version = "0.1.0" cid.workspace = true [dev-dependencies] +rand = { workspace = true, features = ["std", "std_rng"] } [lints] workspace = true diff --git a/primitives/shared/src/commitment/mod.rs b/primitives/shared/src/commitment/mod.rs index 370ff3896..fa9981199 100644 --- a/primitives/shared/src/commitment/mod.rs +++ b/primitives/shared/src/commitment/mod.rs @@ -26,7 +26,7 @@ pub const SHA2_256_TRUNC254_PADDED: u64 = 0x1012; /// https://github.com/multiformats/multicodec/blob/badcfe56bb7e0bbb06b60d57565186cd6be1f932/table.csv#L537 pub const POSEIDON_BLS12_381_A1_FC1: u64 = 0xb401; -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum CommitmentKind { // CommP - Piece commitment Piece, @@ -52,7 +52,7 @@ impl CommitmentKind { } } -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Commitment { commitment: [u8; 32], kind: CommitmentKind, @@ -84,6 +84,10 @@ impl Commitment { if multicodec != FIL_COMMITMENT_SEALED { return Err("invalid multicodec for commitment"); } + + if multihash != POSEIDON_BLS12_381_A1_FC1 { + return Err("invalid multihash for commitment"); + } } } @@ -111,3 +115,117 @@ pub fn zero_piece_commitment(size: PaddedPieceSize) -> Commitment { kind: CommitmentKind::Piece, } } + +#[cfg(test)] +mod tests { + use cid::{multihash::Multihash, Cid}; + + // use rand::thread_rng; + use crate::commitment::{ + Commitment, CommitmentKind, FIL_COMMITMENT_SEALED, FIL_COMMITMENT_UNSEALED, + POSEIDON_BLS12_381_A1_FC1, SHA2_256_TRUNC254_PADDED, + }; + + fn rand_comm() -> [u8; 32] { + rand::random::<[u8; 32]>() + } + + #[test] + fn comm_d_to_cid() { + let comm = rand_comm(); + + let cid = Commitment::new(comm, CommitmentKind::Data).cid(); + assert_eq!(cid.codec(), FIL_COMMITMENT_UNSEALED); + assert_eq!(cid.hash().code(), SHA2_256_TRUNC254_PADDED); + assert_eq!(cid.hash().digest(), comm); + } + + #[test] + fn cid_to_comm_d() { + let comm = rand_comm(); + + // Correct hash format + let mh = Multihash::wrap(SHA2_256_TRUNC254_PADDED, &comm).unwrap(); + let c = Cid::new_v1(FIL_COMMITMENT_UNSEALED, mh); + let commitment = Commitment::from_cid(&c, CommitmentKind::Data).unwrap(); + assert_eq!(commitment.raw(), comm); + + // Should fail with incorrect codec + let c = Cid::new_v1(FIL_COMMITMENT_SEALED, mh); + let commitment = Commitment::from_cid(&c, CommitmentKind::Data); + assert!(commitment.is_err()); + + // Incorrect hash format + let mh = Multihash::wrap(0x9999, &comm).unwrap(); + let c = Cid::new_v1(FIL_COMMITMENT_UNSEALED, mh); + let commitment = Commitment::from_cid(&c, CommitmentKind::Data); + assert!(commitment.is_err()); + } + + #[test] + fn comm_r_to_cid() { + let comm = rand_comm(); + let cid = Commitment::new(comm, CommitmentKind::Replica).cid(); + + assert_eq!(cid.codec(), FIL_COMMITMENT_SEALED); + assert_eq!(cid.hash().code(), POSEIDON_BLS12_381_A1_FC1); + assert_eq!(cid.hash().digest(), comm); + } + + #[test] + fn cid_to_comm_r() { + let comm = rand_comm(); + + // Correct hash format + let mh = Multihash::wrap(POSEIDON_BLS12_381_A1_FC1, &comm).unwrap(); + let c = Cid::new_v1(FIL_COMMITMENT_SEALED, mh); + let commitment = Commitment::from_cid(&c, CommitmentKind::Replica).unwrap(); + assert_eq!(commitment.raw(), comm); + + // Should fail with incorrect codec + let c = Cid::new_v1(FIL_COMMITMENT_UNSEALED, mh); + let commitment = Commitment::from_cid(&c, CommitmentKind::Replica); + assert!(commitment.is_err()); + + // Incorrect hash format + let mh = Multihash::wrap(0x9999, &comm).unwrap(); + let c = Cid::new_v1(FIL_COMMITMENT_SEALED, mh); + let commitment = Commitment::from_cid(&c, CommitmentKind::Replica); + assert!(commitment.is_err()); + } + + #[test] + fn symmetric_conversion() { + let comm = rand_comm(); + + // piece + let cid = Commitment::new(comm, CommitmentKind::Piece).cid(); + assert_eq!( + Commitment::from_cid(&cid, CommitmentKind::Piece).unwrap(), + Commitment { + commitment: comm, + kind: CommitmentKind::Piece + } + ); + + // data + let cid = Commitment::new(comm, CommitmentKind::Data).cid(); + assert_eq!( + Commitment::from_cid(&cid, CommitmentKind::Data).unwrap(), + Commitment { + commitment: comm, + kind: CommitmentKind::Data + } + ); + + // replica + let cid = Commitment::new(comm, CommitmentKind::Replica).cid(); + assert_eq!( + Commitment::from_cid(&cid, CommitmentKind::Replica).unwrap(), + Commitment { + commitment: comm, + kind: CommitmentKind::Replica + } + ); + } +} From 729015936b0eb3a073fcfb0cefcdd7032287375d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20=C4=8Cerni=C4=8D?= Date: Sat, 5 Oct 2024 00:36:55 +0200 Subject: [PATCH 10/24] finaly --- .../src/commands/utils/commp.rs | 25 +- pallets/market/src/commd.rs | 244 +++++++++--------- primitives/shared/src/commitment/mod.rs | 3 +- primitives/shared/src/commitment/zero.rs | 76 +++++- primitives/shared/src/piece.rs | 3 + 5 files changed, 210 insertions(+), 141 deletions(-) diff --git a/cli/polka-storage-provider/src/commands/utils/commp.rs b/cli/polka-storage-provider/src/commands/utils/commp.rs index 4dfc9c0f7..895a91b15 100644 --- a/cli/polka-storage-provider/src/commands/utils/commp.rs +++ b/cli/polka-storage-provider/src/commands/utils/commp.rs @@ -105,8 +105,9 @@ pub enum CommPError { #[cfg(test)] mod tests { - use std::io::Read; + use std::io::{Cursor, Read}; + use primitives_proofs::SectorSize; use primitives_shared::piece::PaddedPieceSize; use crate::commands::utils::commp::{calculate_piece_commitment, ZeroPaddingReader}; @@ -138,8 +139,6 @@ mod tests { #[test] fn test_calculate_piece_commitment() { - use std::io::Cursor; - let data_size: u64 = 200; let data = vec![2u8; data_size as usize]; let cursor = Cursor::new(data.clone()); @@ -156,4 +155,24 @@ mod tests { ] ); } + + #[test] + fn test_zero_piece_commitment() { + let size = SectorSize::_2KiB; + let padded_piece_size = PaddedPieceSize::new(size.bytes()).unwrap(); + let cursor = Cursor::new(vec![]); + let zero_padding_reader = ZeroPaddingReader::new(cursor, *padded_piece_size); + + let commitment = + calculate_piece_commitment(zero_padding_reader, padded_piece_size).unwrap(); + dbg!(commitment.raw()); + + assert_eq!( + commitment.raw(), + [ + 252, 126, 146, 130, 150, 229, 22, 250, 173, 233, 134, 178, 143, 146, 212, 74, 79, + 36, 185, 53, 72, 82, 35, 55, 106, 121, 144, 39, 188, 24, 248, 51 + ] + ); + } } diff --git a/pallets/market/src/commd.rs b/pallets/market/src/commd.rs index 623fda067..61b4fe4b3 100644 --- a/pallets/market/src/commd.rs +++ b/pallets/market/src/commd.rs @@ -4,7 +4,7 @@ use alloc::vec::Vec; use primitives_proofs::SectorSize; use primitives_shared::{ commitment::{zero_piece_commitment, Commitment, CommitmentKind}, - piece::PaddedPieceSize, + piece::{PaddedPieceSize, UnpaddedPieceSize}, NODE_SIZE, }; use sha2::{Digest, Sha256}; @@ -14,12 +14,20 @@ fn ensure_piece_sizes( sector_size: SectorSize, piece_infos: &[PieceInfo], ) -> Result<(), CommDError> { + // Sector should be able to hold all pieces let size_sum = piece_infos.iter().map(|piece| *piece.size).sum::(); - if size_sum > sector_size.bytes() { return Err(CommDError::PieceSizeTooLarge); } + // Check if there are too many pieces for a sector of this size + let sector_size = PaddedPieceSize::new(sector_size.bytes()).unwrap(); + let num_of_pieces = piece_infos.len() as u64; + let max_pieces = *sector_size.unpadded() / *UnpaddedPieceSize::MIN; + if num_of_pieces > max_pieces { + return Err(CommDError::TooManyPieces); + } + Ok(()) } @@ -30,7 +38,8 @@ pub fn compute_unsealed_sector_commitment( ) -> Result { let padded_sector_size = PaddedPieceSize::new(sector_size.bytes()).unwrap(); - // In case of no pieces, return the zero commitment for the whole sector. + // In case of no pieces, return the piece zero commitment for the whole + // sector size. if piece_infos.is_empty() { return Ok(zero_piece_commitment(padded_sector_size)); } @@ -38,40 +47,12 @@ pub fn compute_unsealed_sector_commitment( // Check if pieces are correct sizes. ensure_piece_sizes(sector_size, piece_infos)?; - // Stack used for piece reduction - let mut stack = Stack::new(); - - // We first load the first piece into the stack. That first element is - // removed from the stack - let first = piece_infos.first().expect("unreachable: !is_empty()"); - // Push the first element on the stack - stack.shift(*first); - - // Iterate all pieces - for piece_info in piece_infos.iter().skip(1) { - // Check if the last element of the stack is smaller than the current - // piece. If it is, add a padding piece to the stack. This is needed - // because when we reduce two pieces they have to be the same size. - while *stack.peek().size < *piece_info.size { - stack.shift_reduce(zero_padding(stack.peek().size)?)? - } - - // Push the actual piece onto the stack - stack.shift_reduce(*piece_info)?; - } - - // If we had odd number of pieces. add a 0 padded piece to the end of the stack - while stack.len() > 1 { - stack.shift_reduce(zero_padding(stack.peek().size)?)?; - } - - // Stacks size must be 1. - if stack.len() != 1 { - return Err(CommDError::InvalidStackSize); - } - - // Last element on the stack is the actual comm_d of the piece - let commitment = stack.pop()?.commitment; + // Reduce the pieces to the 1-piece commitment + let mut reduction = CommDPieceReduction::new(); + piece_infos.iter().for_each(|piece_info| { + reduction.add_piece(*piece_info); + }); + let commitment = reduction.finish().expect("at least one piece was added"); Ok(commitment) } @@ -84,95 +65,113 @@ pub struct PieceInfo { pub size: PaddedPieceSize, } -/// Stack used for piece reduction. -/// TODO: Temporary implementation copied from the filecoin -struct Stack(Vec); +struct CommDPieceReduction { + /// Pieces in queue that will be reduced + pieces: Vec, +} -impl Stack { - /// Creates a new stack. +impl CommDPieceReduction { fn new() -> Self { - Stack(Vec::new()) - } - - /// Pushes a single element onto the stack. - fn shift(&mut self, el: PieceInfo) { - self.0.push(el) - } - - /// Look at the last element of the stack. - fn peek(&self) -> &PieceInfo { - &self.0[self.0.len() - 1] - } - - /// Look at the second to last element of the stack. - fn peek2(&self) -> &PieceInfo { - &self.0[self.0.len() - 2] - } - - /// Pop the last element of the stack. - fn pop(&mut self) -> Result { - Ok(self.0.pop().unwrap()) + CommDPieceReduction { pieces: Vec::new() } } - fn reduce1(&mut self) -> Result { - if self.len() < 2 { - return Ok(false); + // Add new piece + fn add_piece(&mut self, piece: PieceInfo) { + // Handle first piece + if self.pieces.is_empty() { + self.pieces.push(piece); + return; } - if self.peek().size == self.peek2().size { - let right = self.pop()?; - let left = self.pop()?; - let joined = join_piece_infos(left, right)?; - self.shift(joined); - return Ok(true); + // Add padding pieces to the stack until we reduce the current pieces to + // the size that is equal to the new piece. With this we achieve that + // the new piece will be reduced to a single piece after adding it to + // the stack. + loop { + let last_added_piece_size = self.pieces.last().expect("at least one piece exists").size; + if *last_added_piece_size >= *piece.size { + break; + } + + let padding_piece = padding_piece(last_added_piece_size); + self.pieces.push(padding_piece); + + // We need to reduce the pieces before the next iteration. Because + // we are always adding the padding to the last piece. And the last + // piece changes based on the result of reduction. + self.reduce(); } - Ok(false) - } - - fn reduce(&mut self) -> Result<(), CommDError> { - while self.reduce1()? {} - Ok(()) - } + // Add the new piece to the queue + self.pieces.push(piece); - fn shift_reduce(&mut self, piece: PieceInfo) -> Result<(), CommDError> { - self.shift(piece); - self.reduce() + // Reduce the pieces + self.reduce(); } - fn len(&self) -> usize { - self.0.len() + /// Combine pieces until there are any in the queue available to combine + fn reduce(&mut self) { + loop { + // If there is only a single piece on the stack we break the loop + let pieces_len = self.pieces.len(); + if pieces_len < 2 { + break; + } + + // If the two pieces on top of the stack are not the same size, we + // can't reduce them + let last_piece_size = self.pieces[pieces_len - 1].size; + let second_last_piece_size = self.pieces[pieces_len - 2].size; + if last_piece_size != second_last_piece_size { + break; + } + + // Pop and join the two pieces on top of the stack. Push the + // combined piece back to the stack + let last_piece = self + .pieces + .pop() + .expect("we know there are at least two pieces"); + let second_last_piece = self + .pieces + .pop() + .expect("we know there are at least two pieces"); + let joined = + join_piece_infos(second_last_piece, last_piece).expect("pieces are the same size"); + self.pieces.push(joined); + } } -} - -/// Create a padding `PieceInfo` of size `size`. -pub fn zero_padding(piece_size: PaddedPieceSize) -> Result { - let mut commitment = [0u8; 32]; - // TODO: cache common piece hashes - let mut hashed_size = 64; - let h1 = piece_hash(&commitment, &commitment); - commitment.copy_from_slice(h1.as_ref()); - - while hashed_size < *piece_size { - let h = piece_hash(&commitment, &commitment); - commitment.copy_from_slice(h.as_ref()); - hashed_size *= 2; - } + /// Finish the reduction of all pieces. Result is a data commitment for the + /// pieces added. + fn finish(mut self) -> Option { + // Check if we still have more then one piece on the stack. If we do, + // that means that we should add some additional padding pieces at the + // end until we can reduce them to a single piece + while self.pieces.len() > 1 { + let last_added_piece_size = self.pieces.last().expect("at least one piece exists").size; + self.pieces.push(padding_piece(last_added_piece_size)); + self.reduce(); + } - if hashed_size != *piece_size { - return Err(CommDError::InvalidPieceSize); + // Finally a single piece with the commitment that represents all + // reduced pieces + Some(self.pieces.pop()?.commitment) } +} - Ok(PieceInfo { - commitment: Commitment::new(commitment, CommitmentKind::Piece), +/// Create a piece of specific size used as a padding. +fn padding_piece(piece_size: PaddedPieceSize) -> PieceInfo { + PieceInfo { + commitment: zero_piece_commitment(piece_size), size: piece_size, - }) + } } /// Join two equally sized `PieceInfo`s together, by hashing them and adding /// their sizes. fn join_piece_infos(left: PieceInfo, right: PieceInfo) -> Result { + // The pieces passed should be same size if left.size != right.size { return Err(CommDError::InvalidPieceSize); } @@ -189,22 +188,27 @@ fn join_piece_infos(left: PieceInfo, right: PieceInfo) -> Result Vec { +/// Calculate Hash of two raw piece commitments +pub fn piece_hash(a: &[u8], b: &[u8]) -> [u8; 32] { let mut buf = [0u8; NODE_SIZE * 2]; buf[..NODE_SIZE].copy_from_slice(a); buf[NODE_SIZE..].copy_from_slice(b); - let mut hasher = Sha256::new(); - hasher.update(buf); - hasher.finalize().to_vec() + let hashed = Sha256::digest(buf); + let mut result = [0u8; 32]; + result.copy_from_slice(&hashed); + + // strip last two bits, to ensure result is in Fr. + result[31] &= 0b0011_1111; + + result } #[derive(Debug)] pub enum CommDError { - PieceSizeTooLarge, InvalidPieceSize, - InvalidStackSize, + PieceSizeTooLarge, + TooManyPieces, } #[cfg(test)] @@ -227,24 +231,6 @@ mod tests { 36, 185, 53, 72, 82, 35, 55, 106, 121, 144, 39, 188, 24, 248, 51 ] ); - - let comm_d = zero_piece_commitment(PaddedPieceSize::new(2048).unwrap()); - assert_eq!( - comm_d.raw(), - [ - 252, 126, 146, 130, 150, 229, 22, 250, 173, 233, 134, 178, 143, 146, 212, 74, 79, - 36, 185, 53, 72, 82, 35, 55, 106, 121, 144, 39, 188, 24, 248, 51 - ] - ); - - let comm_d = zero_piece_commitment(PaddedPieceSize::new(128).unwrap()); - assert_eq!( - comm_d.raw(), - [ - 55, 49, 187, 153, 172, 104, 159, 102, 238, 245, 151, 62, 74, 148, 218, 24, 143, 77, - 220, 174, 88, 7, 36, 252, 111, 63, 214, 13, 253, 72, 131, 51 - ] - ); } /// Reference: diff --git a/primitives/shared/src/commitment/mod.rs b/primitives/shared/src/commitment/mod.rs index fa9981199..f0c2cc092 100644 --- a/primitives/shared/src/commitment/mod.rs +++ b/primitives/shared/src/commitment/mod.rs @@ -103,7 +103,8 @@ impl Commitment { pub fn cid(&self) -> Cid { let multihash = self.kind.multihash(); let multicodec = self.kind.multicodec(); - let hash = Multihash::wrap(multihash, &self.commitment).expect("correct commitment"); + let hash = Multihash::wrap(multihash, &self.commitment) + .expect("multihash is large enough so it can wrap the commitment"); Cid::new_v1(multicodec, hash) } } diff --git a/primitives/shared/src/commitment/zero.rs b/primitives/shared/src/commitment/zero.rs index 973417101..fa919d95a 100644 --- a/primitives/shared/src/commitment/zero.rs +++ b/primitives/shared/src/commitment/zero.rs @@ -1,189 +1,249 @@ use crate::piece::PaddedPieceSize; -const LEVELS: usize = 37; -const SKIP: usize = 2; // can't generate for 32, 64b +/// Can't generate for: 1, 2, 4, 8, 16, 32, 64 bytes +const SKIP: u32 = 7; -/// Returns a zero piece commitment based on the amount of space needed to pad. -pub fn zero_piece_commitment(sz: PaddedPieceSize) -> [u8; 32] { - let level: usize = sz.trailing_zeros() as usize - SKIP - 5; // 2^5 = 32 - PIECE_COMMS[level] +/// Returns a zero piece commitment for a specified piece size +pub fn zero_piece_commitment(size: PaddedPieceSize) -> [u8; 32] { + let level = size.trailing_zeros() - SKIP; + PIECE_COMMS[level as usize] } -/// Zero piece commitments. This is statically defined to be able to pad remaining space with. -const PIECE_COMMS: [[u8; 32]; LEVELS - SKIP] = [ +/// Zero piece commitments. This is statically defined to be able to pad +/// remaining space with. +const PIECE_COMMS: [[u8; 32]; 35] = [ + // 128 bytes [ 0x37, 0x31, 0xbb, 0x99, 0xac, 0x68, 0x9f, 0x66, 0xee, 0xf5, 0x97, 0x3e, 0x4a, 0x94, 0xda, 0x18, 0x8f, 0x4d, 0xdc, 0xae, 0x58, 0x7, 0x24, 0xfc, 0x6f, 0x3f, 0xd6, 0xd, 0xfd, 0x48, 0x83, 0x33, ], + // 256 bytes [ 0x64, 0x2a, 0x60, 0x7e, 0xf8, 0x86, 0xb0, 0x4, 0xbf, 0x2c, 0x19, 0x78, 0x46, 0x3a, 0xe1, 0xd4, 0x69, 0x3a, 0xc0, 0xf4, 0x10, 0xeb, 0x2d, 0x1b, 0x7a, 0x47, 0xfe, 0x20, 0x5e, 0x5e, 0x75, 0xf, ], + // 512 bytes [ 0x57, 0xa2, 0x38, 0x1a, 0x28, 0x65, 0x2b, 0xf4, 0x7f, 0x6b, 0xef, 0x7a, 0xca, 0x67, 0x9b, 0xe4, 0xae, 0xde, 0x58, 0x71, 0xab, 0x5c, 0xf3, 0xeb, 0x2c, 0x8, 0x11, 0x44, 0x88, 0xcb, 0x85, 0x26, ], + // 1024 bytes [ 0x1f, 0x7a, 0xc9, 0x59, 0x55, 0x10, 0xe0, 0x9e, 0xa4, 0x1c, 0x46, 0xb, 0x17, 0x64, 0x30, 0xbb, 0x32, 0x2c, 0xd6, 0xfb, 0x41, 0x2e, 0xc5, 0x7c, 0xb1, 0x7d, 0x98, 0x9a, 0x43, 0x10, 0x37, 0x2f, ], + // 2048 bytes [ 0xfc, 0x7e, 0x92, 0x82, 0x96, 0xe5, 0x16, 0xfa, 0xad, 0xe9, 0x86, 0xb2, 0x8f, 0x92, 0xd4, 0x4a, 0x4f, 0x24, 0xb9, 0x35, 0x48, 0x52, 0x23, 0x37, 0x6a, 0x79, 0x90, 0x27, 0xbc, 0x18, 0xf8, 0x33, ], + // 4096 bytes [ 0x8, 0xc4, 0x7b, 0x38, 0xee, 0x13, 0xbc, 0x43, 0xf4, 0x1b, 0x91, 0x5c, 0xe, 0xed, 0x99, 0x11, 0xa2, 0x60, 0x86, 0xb3, 0xed, 0x62, 0x40, 0x1b, 0xf9, 0xd5, 0x8b, 0x8d, 0x19, 0xdf, 0xf6, 0x24, ], + // 8192 bytes [ 0xb2, 0xe4, 0x7b, 0xfb, 0x11, 0xfa, 0xcd, 0x94, 0x1f, 0x62, 0xaf, 0x5c, 0x75, 0xf, 0x3e, 0xa5, 0xcc, 0x4d, 0xf5, 0x17, 0xd5, 0xc4, 0xf1, 0x6d, 0xb2, 0xb4, 0xd7, 0x7b, 0xae, 0xc1, 0xa3, 0x2f, ], + // 16384 bytes = 16 KiB [ 0xf9, 0x22, 0x61, 0x60, 0xc8, 0xf9, 0x27, 0xbf, 0xdc, 0xc4, 0x18, 0xcd, 0xf2, 0x3, 0x49, 0x31, 0x46, 0x0, 0x8e, 0xae, 0xfb, 0x7d, 0x2, 0x19, 0x4d, 0x5e, 0x54, 0x81, 0x89, 0x0, 0x51, 0x8, ], + // 32768 bytes = 32 KiB [ 0x2c, 0x1a, 0x96, 0x4b, 0xb9, 0xb, 0x59, 0xeb, 0xfe, 0xf, 0x6d, 0xa2, 0x9a, 0xd6, 0x5a, 0xe3, 0xe4, 0x17, 0x72, 0x4a, 0x8f, 0x7c, 0x11, 0x74, 0x5a, 0x40, 0xca, 0xc1, 0xe5, 0xe7, 0x40, 0x11, ], + // 65536 bytes = 64 KiB [ 0xfe, 0xe3, 0x78, 0xce, 0xf1, 0x64, 0x4, 0xb1, 0x99, 0xed, 0xe0, 0xb1, 0x3e, 0x11, 0xb6, 0x24, 0xff, 0x9d, 0x78, 0x4f, 0xbb, 0xed, 0x87, 0x8d, 0x83, 0x29, 0x7e, 0x79, 0x5e, 0x2, 0x4f, 0x2, ], + // 131072 bytes = 128 KiB [ 0x8e, 0x9e, 0x24, 0x3, 0xfa, 0x88, 0x4c, 0xf6, 0x23, 0x7f, 0x60, 0xdf, 0x25, 0xf8, 0x3e, 0xe4, 0xd, 0xca, 0x9e, 0xd8, 0x79, 0xeb, 0x6f, 0x63, 0x52, 0xd1, 0x50, 0x84, 0xf5, 0xad, 0xd, 0x3f, ], + // 262144 bytes = 256 KiB [ 0x75, 0x2d, 0x96, 0x93, 0xfa, 0x16, 0x75, 0x24, 0x39, 0x54, 0x76, 0xe3, 0x17, 0xa9, 0x85, 0x80, 0xf0, 0x9, 0x47, 0xaf, 0xb7, 0xa3, 0x5, 0x40, 0xd6, 0x25, 0xa9, 0x29, 0x1c, 0xc1, 0x2a, 0x7, ], + // 524288 bytes = 512 KiB [ 0x70, 0x22, 0xf6, 0xf, 0x7e, 0xf6, 0xad, 0xfa, 0x17, 0x11, 0x7a, 0x52, 0x61, 0x9e, 0x30, 0xce, 0xa8, 0x2c, 0x68, 0x7, 0x5a, 0xdf, 0x1c, 0x66, 0x77, 0x86, 0xec, 0x50, 0x6e, 0xef, 0x2d, 0x19, ], + // 1048576 bytes = 1 MiB [ 0xd9, 0x98, 0x87, 0xb9, 0x73, 0x57, 0x3a, 0x96, 0xe1, 0x13, 0x93, 0x64, 0x52, 0x36, 0xc1, 0x7b, 0x1f, 0x4c, 0x70, 0x34, 0xd7, 0x23, 0xc7, 0xa9, 0x9f, 0x70, 0x9b, 0xb4, 0xda, 0x61, 0x16, 0x2b, ], + // 2097152 bytes = 2 MiB [ 0xd0, 0xb5, 0x30, 0xdb, 0xb0, 0xb4, 0xf2, 0x5c, 0x5d, 0x2f, 0x2a, 0x28, 0xdf, 0xee, 0x80, 0x8b, 0x53, 0x41, 0x2a, 0x2, 0x93, 0x1f, 0x18, 0xc4, 0x99, 0xf5, 0xa2, 0x54, 0x8, 0x6b, 0x13, 0x26, ], + // 4194304 bytes = 4 MiB [ 0x84, 0xc0, 0x42, 0x1b, 0xa0, 0x68, 0x5a, 0x1, 0xbf, 0x79, 0x5a, 0x23, 0x44, 0x6, 0x4f, 0xe4, 0x24, 0xbd, 0x52, 0xa9, 0xd2, 0x43, 0x77, 0xb3, 0x94, 0xff, 0x4c, 0x4b, 0x45, 0x68, 0xe8, 0x11, ], + // 8388608 bytes = 8 MiB [ 0x65, 0xf2, 0x9e, 0x5d, 0x98, 0xd2, 0x46, 0xc3, 0x8b, 0x38, 0x8c, 0xfc, 0x6, 0xdb, 0x1f, 0x6b, 0x2, 0x13, 0x3, 0xc5, 0xa2, 0x89, 0x0, 0xb, 0xdc, 0xe8, 0x32, 0xa9, 0xc3, 0xec, 0x42, 0x1c, ], + // 16777216 bytes = 16 MiB [ 0xa2, 0x24, 0x75, 0x8, 0x28, 0x58, 0x50, 0x96, 0x5b, 0x7e, 0x33, 0x4b, 0x31, 0x27, 0xb0, 0xc0, 0x42, 0xb1, 0xd0, 0x46, 0xdc, 0x54, 0x40, 0x21, 0x37, 0x62, 0x7c, 0xd8, 0x79, 0x9c, 0xe1, 0x3a, ], + // 33554432 bytes = 32 MiB [ 0xda, 0xfd, 0xab, 0x6d, 0xa9, 0x36, 0x44, 0x53, 0xc2, 0x6d, 0x33, 0x72, 0x6b, 0x9f, 0xef, 0xe3, 0x43, 0xbe, 0x8f, 0x81, 0x64, 0x9e, 0xc0, 0x9, 0xaa, 0xd3, 0xfa, 0xff, 0x50, 0x61, 0x75, 0x8, ], + // 67108864 bytes = 64 MiB [ 0xd9, 0x41, 0xd5, 0xe0, 0xd6, 0x31, 0x4a, 0x99, 0x5c, 0x33, 0xff, 0xbd, 0x4f, 0xbe, 0x69, 0x11, 0x8d, 0x73, 0xd4, 0xe5, 0xfd, 0x2c, 0xd3, 0x1f, 0xf, 0x7c, 0x86, 0xeb, 0xdd, 0x14, 0xe7, 0x6, ], + // 134217728 bytes = 128 MiB [ 0x51, 0x4c, 0x43, 0x5c, 0x3d, 0x4, 0xd3, 0x49, 0xa5, 0x36, 0x5f, 0xbd, 0x59, 0xff, 0xc7, 0x13, 0x62, 0x91, 0x11, 0x78, 0x59, 0x91, 0xc1, 0xa3, 0xc5, 0x3a, 0xf2, 0x20, 0x79, 0x74, 0x1a, 0x2f, ], + // 268435456 bytes = 256 MiB [ 0xad, 0x6, 0x85, 0x39, 0x69, 0xd3, 0x7d, 0x34, 0xff, 0x8, 0xe0, 0x9f, 0x56, 0x93, 0xa, 0x4a, 0xd1, 0x9a, 0x89, 0xde, 0xf6, 0xc, 0xbf, 0xee, 0x7e, 0x1d, 0x33, 0x81, 0xc1, 0xe7, 0x1c, 0x37, ], + // 536870912 bytes = 512 MiB [ 0x39, 0x56, 0xe, 0x7b, 0x13, 0xa9, 0x3b, 0x7, 0xa2, 0x43, 0xfd, 0x27, 0x20, 0xff, 0xa7, 0xcb, 0x3e, 0x1d, 0x2e, 0x50, 0x5a, 0xb3, 0x62, 0x9e, 0x79, 0xf4, 0x63, 0x13, 0x51, 0x2c, 0xda, 0x6, ], + // 1073741824 bytes = 1024 MiB = 1 GiB [ 0xcc, 0xc3, 0xc0, 0x12, 0xf5, 0xb0, 0x5e, 0x81, 0x1a, 0x2b, 0xbf, 0xdd, 0xf, 0x68, 0x33, 0xb8, 0x42, 0x75, 0xb4, 0x7b, 0xf2, 0x29, 0xc0, 0x5, 0x2a, 0x82, 0x48, 0x4f, 0x3c, 0x1a, 0x5b, 0x3d, ], + // 2147483648 bytes = 2048 MiB = 2 GiB [ 0x7d, 0xf2, 0x9b, 0x69, 0x77, 0x31, 0x99, 0xe8, 0xf2, 0xb4, 0xb, 0x77, 0x91, 0x9d, 0x4, 0x85, 0x9, 0xee, 0xd7, 0x68, 0xe2, 0xc7, 0x29, 0x7b, 0x1f, 0x14, 0x37, 0x3, 0x4f, 0xc3, 0xc6, 0x2c, ], + // 4294967296 bytes = 4096 MiB = 4 GiB [ 0x66, 0xce, 0x5, 0xa3, 0x66, 0x75, 0x52, 0xcf, 0x45, 0xc0, 0x2b, 0xcc, 0x4e, 0x83, 0x92, 0x91, 0x9b, 0xde, 0xac, 0x35, 0xde, 0x2f, 0xf5, 0x62, 0x71, 0x84, 0x8e, 0x9f, 0x7b, 0x67, 0x51, 0x7, ], + // 8589934592 bytes = 8192 MiB = 8 GiB [ 0xd8, 0x61, 0x2, 0x18, 0x42, 0x5a, 0xb5, 0xe9, 0x5b, 0x1c, 0xa6, 0x23, 0x9d, 0x29, 0xa2, 0xe4, 0x20, 0xd7, 0x6, 0xa9, 0x6f, 0x37, 0x3e, 0x2f, 0x9c, 0x9a, 0x91, 0xd7, 0x59, 0xd1, 0x9b, 0x1, ], + // 17179869184 bytes = 16384 MiB = 16 GiB [ 0x6d, 0x36, 0x4b, 0x1e, 0xf8, 0x46, 0x44, 0x1a, 0x5a, 0x4a, 0x68, 0x86, 0x23, 0x14, 0xac, 0xc0, 0xa4, 0x6f, 0x1, 0x67, 0x17, 0xe5, 0x34, 0x43, 0xe8, 0x39, 0xee, 0xdf, 0x83, 0xc2, 0x85, 0x3c, ], + // 34359738368 bytes = 32768 MiB = 32 GiB [ 0x7, 0x7e, 0x5f, 0xde, 0x35, 0xc5, 0xa, 0x93, 0x3, 0xa5, 0x50, 0x9, 0xe3, 0x49, 0x8a, 0x4e, 0xbe, 0xdf, 0xf3, 0x9c, 0x42, 0xb7, 0x10, 0xb7, 0x30, 0xd8, 0xec, 0x7a, 0xc7, 0xaf, 0xa6, 0x3e, ], + // 68719476736 bytes = 65536 MiB = 64 GiB [ 0xe6, 0x40, 0x5, 0xa6, 0xbf, 0xe3, 0x77, 0x79, 0x53, 0xb8, 0xad, 0x6e, 0xf9, 0x3f, 0xf, 0xca, 0x10, 0x49, 0xb2, 0x4, 0x16, 0x54, 0xf2, 0xa4, 0x11, 0xf7, 0x70, 0x27, 0x99, 0xce, 0xce, 0x2, ], + // 137438953472 bytes = 131072 MiB = 128 GiB [ 0x25, 0x9d, 0x3d, 0x6b, 0x1f, 0x4d, 0x87, 0x6d, 0x11, 0x85, 0xe1, 0x12, 0x3a, 0xf6, 0xf5, 0x50, 0x1a, 0xf0, 0xf6, 0x7c, 0xf1, 0x5b, 0x52, 0x16, 0x25, 0x5b, 0x7b, 0x17, 0x8d, 0x12, 0x5, 0x1d, ], + // 274877906944 bytes = 262144 MiB = 256 GiB [ 0x3f, 0x9a, 0x4d, 0x41, 0x1d, 0xa4, 0xef, 0x1b, 0x36, 0xf3, 0x5f, 0xf0, 0xa1, 0x95, 0xae, 0x39, 0x2a, 0xb2, 0x3f, 0xee, 0x79, 0x67, 0xb7, 0xc4, 0x1b, 0x3, 0xd1, 0x61, 0x3f, 0xc2, 0x92, 0x39, ], + // 549755813888 bytes = 524288 MiB = 512 GiB [ 0xfe, 0x4e, 0xf3, 0x28, 0xc6, 0x1a, 0xa3, 0x9c, 0xfd, 0xb2, 0x48, 0x4e, 0xaa, 0x32, 0xa1, 0x51, 0xb1, 0xfe, 0x3d, 0xfd, 0x1f, 0x96, 0xdd, 0x8c, 0x97, 0x11, 0xfd, 0x86, 0xd6, 0xc5, 0x81, 0x13, ], + // 1099511627776 bytes = 1048576 MiB = 1024 GiB [ 0xf5, 0x5d, 0x68, 0x90, 0xe, 0x2d, 0x83, 0x81, 0xec, 0xcb, 0x81, 0x64, 0xcb, 0x99, 0x76, 0xf2, 0x4b, 0x2d, 0xe0, 0xdd, 0x61, 0xa3, 0x1b, 0x97, 0xce, 0x6e, 0xb2, 0x38, 0x50, 0xd5, 0xe8, 0x19, ], + // 2199023255552 bytes = 2097152 MiB = 2048 GiB [ 0xaa, 0xaa, 0x8c, 0x4c, 0xb4, 0xa, 0xac, 0xee, 0x1e, 0x2, 0xdc, 0x65, 0x42, 0x4b, 0x2a, 0x6c, 0x8e, 0x99, 0xf8, 0x3, 0xb7, 0x2f, 0x79, 0x29, 0xc4, 0x10, 0x1d, 0x7f, 0xae, 0x6b, 0xff, 0x32, ], ]; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_zero_piece_commitment() { + assert_eq!( + zero_piece_commitment(PaddedPieceSize::new(2048).unwrap()), + [ + 252, 126, 146, 130, 150, 229, 22, 250, 173, 233, 134, 178, 143, 146, 212, 74, 79, + 36, 185, 53, 72, 82, 35, 55, 106, 121, 144, 39, 188, 24, 248, 51 + ] + ); + + assert_eq!( + zero_piece_commitment(PaddedPieceSize::new(128).unwrap()), + [ + 55, 49, 187, 153, 172, 104, 159, 102, 238, 245, 151, 62, 74, 148, 218, 24, 143, 77, + 220, 174, 88, 7, 36, 252, 111, 63, 214, 13, 253, 72, 131, 51 + ] + ); + } +} diff --git a/primitives/shared/src/piece.rs b/primitives/shared/src/piece.rs index 5deae2eb9..79b3e3508 100644 --- a/primitives/shared/src/piece.rs +++ b/primitives/shared/src/piece.rs @@ -8,6 +8,9 @@ use crate::NODE_SIZE; pub struct UnpaddedPieceSize(u64); impl UnpaddedPieceSize { + /// The minimum pice size + pub const MIN: UnpaddedPieceSize = UnpaddedPieceSize(127); + /// Initialize new unpadded piece size. Error is returned if the size is /// invalid. pub fn new(size: u64) -> Result { From d4e148321ce138aef3d204d95c6b70bc7334decc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20=C4=8Cerni=C4=8D?= Date: Sat, 5 Oct 2024 00:53:24 +0200 Subject: [PATCH 11/24] handle errors --- pallets/market/src/lib.rs | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/pallets/market/src/lib.rs b/pallets/market/src/lib.rs index cf9a22054..a13ec66a0 100644 --- a/pallets/market/src/lib.rs +++ b/pallets/market/src/lib.rs @@ -463,6 +463,8 @@ pub mod pallet { TooManyDealsPerBlock, /// Try to call an operation as a storage provider but the account is not registered as a storage provider. StorageProviderNotRegistered, + /// CommD related error + CommD, } pub enum DealActivationError { @@ -527,6 +529,8 @@ pub mod pallet { DealDurationOutOfBounds, /// Deal's piece_cid is invalid. InvalidPieceCid(cid::Error), + /// CommD related error + CommD(&'static str), } impl core::fmt::Debug for ProposalError { @@ -553,6 +557,9 @@ pub mod pallet { ProposalError::InvalidPieceCid(_err) => { write!(f, "ProposalError::InvalidPieceCid") } + ProposalError::CommD(err) => { + write!(f, "ProposalError::CommD: {}", err) + } } } } @@ -965,15 +972,21 @@ pub mod pallet { let pieces = proposals .into_iter() .map(|p| { - let cid = p.cid().unwrap(); - let commitment = Commitment::from_cid(&cid, CommitmentKind::Piece).unwrap(); + let cid = p.cid()?; + let commitment = Commitment::from_cid(&cid, CommitmentKind::Piece) + .map_err(|err| ProposalError::CommD(err))?; - crate::commd::PieceInfo { + Ok(crate::commd::PieceInfo { size: PaddedPieceSize::new(p.piece_size).unwrap(), commitment, - } + }) }) - .collect::>(); + .collect::, ProposalError>>(); + + let pieces = pieces.map_err(|err| { + log::error!("error occurred while calculating commd: {}", err); + Error::::CommD + })?; let sector_size = sector_type.sector_size(); let comm_d = compute_unsealed_sector_commitment(sector_size, &pieces).unwrap(); From 2d0bf2ae11ef8b73ec39721a6c54359ee8f50465 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20=C4=8Cerni=C4=8D?= Date: Sat, 5 Oct 2024 00:57:23 +0200 Subject: [PATCH 12/24] some more error handling --- pallets/market/src/lib.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/pallets/market/src/lib.rs b/pallets/market/src/lib.rs index a13ec66a0..0abdc1469 100644 --- a/pallets/market/src/lib.rs +++ b/pallets/market/src/lib.rs @@ -975,21 +975,24 @@ pub mod pallet { let cid = p.cid()?; let commitment = Commitment::from_cid(&cid, CommitmentKind::Piece) .map_err(|err| ProposalError::CommD(err))?; + let size = PaddedPieceSize::new(p.piece_size) + .map_err(|err| ProposalError::CommD(err))?; - Ok(crate::commd::PieceInfo { - size: PaddedPieceSize::new(p.piece_size).unwrap(), - commitment, - }) + Ok(crate::commd::PieceInfo { size, commitment }) }) .collect::, ProposalError>>(); let pieces = pieces.map_err(|err| { - log::error!("error occurred while calculating commd: {}", err); + log::error!("error occurred while processing pieces: {:?}", err); Error::::CommD })?; let sector_size = sector_type.sector_size(); - let comm_d = compute_unsealed_sector_commitment(sector_size, &pieces).unwrap(); + let comm_d = + compute_unsealed_sector_commitment(sector_size, &pieces).map_err(|err| { + log::error!("error occurred while computing commd: {:?}", err); + Error::::CommD + })?; Ok(comm_d.cid()) } From 88853113433b49a1855527d96dbb0e5a677537d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20=C4=8Cerni=C4=8D?= Date: Sat, 5 Oct 2024 12:34:43 +0200 Subject: [PATCH 13/24] add multiple pieces to reduction --- pallets/market/src/commd.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pallets/market/src/commd.rs b/pallets/market/src/commd.rs index 61b4fe4b3..7be137714 100644 --- a/pallets/market/src/commd.rs +++ b/pallets/market/src/commd.rs @@ -49,9 +49,7 @@ pub fn compute_unsealed_sector_commitment( // Reduce the pieces to the 1-piece commitment let mut reduction = CommDPieceReduction::new(); - piece_infos.iter().for_each(|piece_info| { - reduction.add_piece(*piece_info); - }); + reduction.add_many_pieces(piece_infos); let commitment = reduction.finish().expect("at least one piece was added"); Ok(commitment) @@ -75,7 +73,12 @@ impl CommDPieceReduction { CommDPieceReduction { pieces: Vec::new() } } - // Add new piece + // Add many pieces pieces + fn add_many_pieces(&mut self, pieces: &[PieceInfo]) { + pieces.iter().for_each(|piece| self.add_piece(*piece)); + } + + // Add a single piece fn add_piece(&mut self, piece: PieceInfo) { // Handle first piece if self.pieces.is_empty() { From 4e41c36eb4d6ace747a16b6688637ae98868f60b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20=C4=8Cerni=C4=8D?= Date: Sun, 6 Oct 2024 22:18:44 +0200 Subject: [PATCH 14/24] small changes --- pallets/market/src/commd.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pallets/market/src/commd.rs b/pallets/market/src/commd.rs index 7be137714..6880b7336 100644 --- a/pallets/market/src/commd.rs +++ b/pallets/market/src/commd.rs @@ -64,7 +64,7 @@ pub struct PieceInfo { } struct CommDPieceReduction { - /// Pieces in queue that will be reduced + /// Pieces stack pieces: Vec, } @@ -105,14 +105,14 @@ impl CommDPieceReduction { self.reduce(); } - // Add the new piece to the queue + // Add the new piece to the stack self.pieces.push(piece); // Reduce the pieces self.reduce(); } - /// Combine pieces until there are any in the queue available to combine + /// Combine pieces until there are any on the stack available to combine fn reduce(&mut self) { loop { // If there is only a single piece on the stack we break the loop From aa9181e98a8e078a7154695671a026dfc638c177 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20=C4=8Cerni=C4=8D?= Date: Mon, 7 Oct 2024 16:00:10 +0200 Subject: [PATCH 15/24] fix tests --- pallets/market/src/mock.rs | 11 ++++++++++- pallets/market/src/test.rs | 23 ++++++++++++----------- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/pallets/market/src/mock.rs b/pallets/market/src/mock.rs index 7338b6bb8..bc125d00a 100644 --- a/pallets/market/src/mock.rs +++ b/pallets/market/src/mock.rs @@ -7,7 +7,7 @@ use frame_support::{ PalletId, }; use frame_system::{self as system, pallet_prelude::BlockNumberFor}; -use multihash_codetable::{Code, MultihashDigest}; +use multihash_codetable::{Code, Multihash, MultihashDigest}; use primitives_proofs::RegisteredPoStProof; use sp_core::Pair; use sp_runtime::{ @@ -115,6 +115,15 @@ pub fn sign(pair: &sp_core::sr25519::Pair, bytes: &[u8]) -> MultiSignature { MultiSignature::Sr25519(pair.sign(bytes)) } +/// Generate dummy cid for piece +pub fn piece_cid(data: &str) -> cid::Cid { + let multihash = 0x1012; + let multicodec = 0xf101; + let hash = Multihash::wrap(multihash, data.as_bytes()) + .expect("multihash is large enough so it can wrap the commitment"); + Cid::new_v1(multicodec, hash) +} + pub fn cid_of(data: &str) -> cid::Cid { Cid::new_v1(CID_CODEC, Code::Blake2b256.digest(data.as_bytes())) } diff --git a/pallets/market/src/test.rs b/pallets/market/src/test.rs index 92e550eee..55df9c92c 100644 --- a/pallets/market/src/test.rs +++ b/pallets/market/src/test.rs @@ -625,7 +625,7 @@ fn verify_deals_for_activation() { assert_eq!( Ok(bounded_vec![ Some( - Cid::from_str("bafk2bzaceajreoxfdcpdvitpvxm7vkpvcimlob5ejebqgqidjkz4qoug4q6zu") + Cid::from_str("baga6ea4seaqgi5lnnv4wi5lnnv4wi5lnnv4wi5lnnv4wi5lnnv4wi5lnnv4wi5i") .unwrap() ), None, @@ -809,11 +809,12 @@ fn activate_deals() { .build() ]; + // Piece cid and commd cid are the same if only one piece is in a deal. let piece_cid = - Cid::from_str("bafk2bzacecg3xxc4f2ql2hreiuy767u6r72ekdz54k7luieknboaakhft5rgk") + Cid::from_str("baga6ea4seaqgi5lnnv4wi5lnnv4wi5lnnv4wi5lnnv4wi5lnnv4wi5lnnv4wi5i") .unwrap(); - let placeholder_commd_cid = - Cid::from_str("bafk2bzaceajreoxfdcpdvitpvxm7vkpvcimlob5ejebqgqidjkz4qoug4q6zu") + let commd_cid = + Cid::from_str("baga6ea4seaqgi5lnnv4wi5lnnv4wi5lnnv4wi5lnnv4wi5lnnv4wi5lnnv4wi5i") .unwrap(); assert_eq!( Ok(bounded_vec![ @@ -821,9 +822,9 @@ fn activate_deals() { active_deals: bounded_vec![ActiveDeal { client: account::(ALICE), piece_cid: piece_cid, - piece_size: 18 + piece_size: 128 }], - unsealed_cid: Some(placeholder_commd_cid), + unsealed_cid: Some(commd_cid), }, ActiveSector { active_deals: bounded_vec![], @@ -863,10 +864,10 @@ fn activate_deals_fails_for_1_sector_but_succeeds_for_others() { ]; let piece_cid = - Cid::from_str("bafk2bzacecg3xxc4f2ql2hreiuy767u6r72ekdz54k7luieknboaakhft5rgk") + Cid::from_str("baga6ea4seaqgi5lnnv4wi5lnnv4wi5lnnv4wi5lnnv4wi5lnnv4wi5lnnv4wi5i") .unwrap(); let placeholder_commd_cid = - Cid::from_str("bafk2bzaceajreoxfdcpdvitpvxm7vkpvcimlob5ejebqgqidjkz4qoug4q6zu") + Cid::from_str("baga6ea4seaqgi5lnnv4wi5lnnv4wi5lnnv4wi5lnnv4wi5lnnv4wi5lnnv4wi5i") .unwrap(); assert_eq!( Ok(bounded_vec![ @@ -874,7 +875,7 @@ fn activate_deals_fails_for_1_sector_but_succeeds_for_others() { active_deals: bounded_vec![ActiveDeal { client: account::(ALICE), piece_cid: piece_cid, - piece_size: 18 + piece_size: 128 }], unsealed_cid: Some(placeholder_commd_cid), }, @@ -1770,11 +1771,11 @@ pub struct DealProposalBuilder { impl> Default for DealProposalBuilder { fn default() -> Self { Self { - piece_cid: cid_of("polka-storage-data") + piece_cid: piece_cid("dummydummydummydummydummydummydu") .to_bytes() .try_into() .expect("hash is always 32 bytes"), - piece_size: 18, + piece_size: 128, client: account::(ALICE), provider: account::(PROVIDER), label: bounded_vec![0xb, 0xe, 0xe, 0xf], From ae8b36fb2f41dd22a4b5af515ac4a4435c39d67a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20=C4=8Cerni=C4=8D?= Date: Tue, 8 Oct 2024 13:32:05 +0200 Subject: [PATCH 16/24] tests --- Cargo.lock | 1 + cli/polka-storage-provider/src/commp.rs | 3 +- pallets/market/src/commd.rs | 1 + pallets/market/src/lib.rs | 1 - pallets/market/src/mock.rs | 19 +++----- pallets/market/src/test.rs | 15 ++++-- pallets/storage-provider/Cargo.toml | 1 + pallets/storage-provider/src/lib.rs | 26 ++++------ pallets/storage-provider/src/tests/mod.rs | 48 +++++++++++-------- .../src/tests/pre_commit_sectors.rs | 27 +++++------ .../src/tests/prove_commit_sectors.rs | 5 ++ primitives/shared/src/piece.rs | 3 ++ 12 files changed, 81 insertions(+), 69 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1783036f6..fca0f13df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8701,6 +8701,7 @@ dependencies = [ "pallet-market", "parity-scale-codec", "primitives-proofs", + "primitives-shared", "rstest", "scale-info", "sp-arithmetic 26.0.0", diff --git a/cli/polka-storage-provider/src/commp.rs b/cli/polka-storage-provider/src/commp.rs index 561ccc3b8..68d0047c7 100644 --- a/cli/polka-storage-provider/src/commp.rs +++ b/cli/polka-storage-provider/src/commp.rs @@ -108,8 +108,7 @@ mod tests { use primitives_proofs::SectorSize; use primitives_shared::piece::PaddedPieceSize; - use super::{calculate_piece_commitment, CommPError, ZeroPaddingReader}; - use crate::commands::utils::commp::{calculate_piece_commitment, ZeroPaddingReader}; + use super::{calculate_piece_commitment, ZeroPaddingReader}; #[test] fn test_zero_padding_reader() { diff --git a/pallets/market/src/commd.rs b/pallets/market/src/commd.rs index 6880b7336..8bcf9727a 100644 --- a/pallets/market/src/commd.rs +++ b/pallets/market/src/commd.rs @@ -63,6 +63,7 @@ pub struct PieceInfo { pub size: PaddedPieceSize, } +/// Reduces the pieces to the data commitment of those pieces. struct CommDPieceReduction { /// Pieces stack pieces: Vec, diff --git a/pallets/market/src/lib.rs b/pallets/market/src/lib.rs index 63e63070c..74d9f964f 100644 --- a/pallets/market/src/lib.rs +++ b/pallets/market/src/lib.rs @@ -53,7 +53,6 @@ pub mod pallet { use crate::commd::compute_unsealed_sector_commitment; - pub const CID_CODEC: u64 = 0x55; pub const LOG_TARGET: &'static str = "runtime::market"; /// Allows to extract Balance of an account via the Config::Currency associated type. diff --git a/pallets/market/src/mock.rs b/pallets/market/src/mock.rs index bc125d00a..3b982f5e1 100644 --- a/pallets/market/src/mock.rs +++ b/pallets/market/src/mock.rs @@ -7,7 +7,7 @@ use frame_support::{ PalletId, }; use frame_system::{self as system, pallet_prelude::BlockNumberFor}; -use multihash_codetable::{Code, Multihash, MultihashDigest}; +use multihash_codetable::{Code, MultihashDigest}; use primitives_proofs::RegisteredPoStProof; use sp_core::Pair; use sp_runtime::{ @@ -15,7 +15,7 @@ use sp_runtime::{ AccountId32, BuildStorage, MultiSignature, MultiSigner, }; -use crate::{self as pallet_market, BalanceOf, ClientDealProposal, DealProposal, CID_CODEC}; +use crate::{self as pallet_market, BalanceOf, ClientDealProposal, DealProposal}; type Block = frame_system::mocking::MockBlock; type BlockNumber = u64; @@ -115,17 +115,12 @@ pub fn sign(pair: &sp_core::sr25519::Pair, bytes: &[u8]) -> MultiSignature { MultiSignature::Sr25519(pair.sign(bytes)) } -/// Generate dummy cid for piece -pub fn piece_cid(data: &str) -> cid::Cid { - let multihash = 0x1012; - let multicodec = 0xf101; - let hash = Multihash::wrap(multihash, data.as_bytes()) - .expect("multihash is large enough so it can wrap the commitment"); - Cid::new_v1(multicodec, hash) -} - +// TODO: Remove this function. The codec and hashing is not correct. This is +// still here because I don't want to make the PR even bigger by changing parts +// of the implementations that are relying on this. pub fn cid_of(data: &str) -> cid::Cid { - Cid::new_v1(CID_CODEC, Code::Blake2b256.digest(data.as_bytes())) + let cid_codec = 0x55; + Cid::new_v1(cid_codec, Code::Blake2b256.digest(data.as_bytes())) } pub(crate) type DealProposalOf = diff --git a/pallets/market/src/test.rs b/pallets/market/src/test.rs index 55df9c92c..5eeffd80a 100644 --- a/pallets/market/src/test.rs +++ b/pallets/market/src/test.rs @@ -12,6 +12,7 @@ use primitives_proofs::{ ActiveDeal, ActiveSector, DealId, Market as MarketTrait, RegisteredSealProof, SectorDeal, MAX_DEALS_PER_SECTOR, }; +use primitives_shared::commitment::Commitment; use sp_core::H256; use sp_runtime::AccountId32; @@ -625,8 +626,10 @@ fn verify_deals_for_activation() { assert_eq!( Ok(bounded_vec![ Some( - Cid::from_str("baga6ea4seaqgi5lnnv4wi5lnnv4wi5lnnv4wi5lnnv4wi5lnnv4wi5lnnv4wi5i") - .unwrap() + Cid::from_str( + "baga6ea4seaqgi5lnnv4wi5lnnv4wi5lnnv4wi5lnnv4wi5lnnv4wi5lnnv4wi5i" + ) + .unwrap() ), None, ]), @@ -1770,8 +1773,14 @@ pub struct DealProposalBuilder { impl> Default for DealProposalBuilder { fn default() -> Self { + let piece_commitment = Commitment::new( + *b"dummydummydummydummydummydummydu", + primitives_shared::commitment::CommitmentKind::Piece, + ); + Self { - piece_cid: piece_cid("dummydummydummydummydummydummydu") + piece_cid: piece_commitment + .cid() .to_bytes() .try_into() .expect("hash is always 32 bytes"), diff --git a/pallets/storage-provider/Cargo.toml b/pallets/storage-provider/Cargo.toml index b4ca14634..2631c6447 100644 --- a/pallets/storage-provider/Cargo.toml +++ b/pallets/storage-provider/Cargo.toml @@ -19,6 +19,7 @@ cid = { workspace = true, features = ["alloc"] } codec = { workspace = true, default-features = false, features = ["derive"] } log = { workspace = true, features = ["kv"] } primitives-proofs = { workspace = true, default-features = false } +primitives-shared = { workspace = true } scale-info = { workspace = true, default-features = false, features = ["derive"] } sp-arithmetic = { workspace = true, default-features = false } sp-core = { workspace = true, default-features = false } diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index d0caeeab1..e12851c13 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -30,9 +30,6 @@ mod storage_provider; #[frame_support::pallet(dev_mode)] pub mod pallet { - pub const CID_CODEC: u64 = 0x55; - /// Sourced from multihash code table - pub const BLAKE2B_MULTIHASH_CODE: u64 = 0xB220; pub(crate) const DECLARATIONS_MAX: u32 = 3000; const LOG_TARGET: &'static str = "runtime::storage_provider"; @@ -41,7 +38,7 @@ pub mod pallet { use alloc::{vec, vec::Vec}; use core::fmt::Debug; - use cid::{Cid, Version}; + use cid::Cid; use codec::{Decode, Encode}; use frame_support::{ dispatch::DispatchResult, @@ -62,6 +59,7 @@ pub mod pallet { Market, RegisteredPoStProof, RegisteredSealProof, SectorNumber, StorageProviderValidation, MAX_SECTORS_PER_CALL, }; + use primitives_shared::commitment::{Commitment, CommitmentKind}; use scale_info::TypeInfo; use sp_arithmetic::traits::Zero; @@ -427,7 +425,7 @@ pub mod pallet { sector.expiration, )?; - let unsealed_cid = validate_cid::(§or.unsealed_cid[..])?; + let unsealed_cid = validate_data_commitment_cid::(§or.unsealed_cid[..])?; let deposit = calculate_pre_commit_deposit::(); let sector_on_chain = SectorPreCommitOnChainInfo::new(sector.clone(), deposit, current_block); @@ -1249,22 +1247,16 @@ pub mod pallet { } // Adapted from filecoin reference here: https://github.com/filecoin-project/builtin-actors/blob/54236ae89880bf4aa89b0dba6d9060c3fd2aacee/actors/miner/src/commd.rs#L51-L56 - fn validate_cid(bytes: &[u8]) -> Result> { - let c = Cid::try_from(bytes).map_err(|e| { + fn validate_data_commitment_cid(bytes: &[u8]) -> Result> { + let cid = Cid::try_from(bytes).map_err(|e| { log::error!(target: LOG_TARGET, e:?; "failed to validate cid"); Error::::InvalidCid })?; - // these values should be consistent with the cid's created by the SP. - // They could change in the future when we make a definitive decision on what hashing algorithm to use and such - ensure!( - c.version() == Version::V1 - && c.codec() == CID_CODEC // The codec should align with our CID_CODEC value. - && c.hash().code() == BLAKE2B_MULTIHASH_CODE // The CID should be hashed using blake2b - && c.hash().size() == 32, - Error::::InvalidCid - ); - Ok(c) + // This checks if the cid represents correct commitment + Commitment::from_cid(&cid, CommitmentKind::Data).map_err(|_| Error::::InvalidCid)?; + + Ok(cid) } /// Calculate the required pre commit deposit amount diff --git a/pallets/storage-provider/src/tests/mod.rs b/pallets/storage-provider/src/tests/mod.rs index 1f31b54b4..369010f0d 100644 --- a/pallets/storage-provider/src/tests/mod.rs +++ b/pallets/storage-provider/src/tests/mod.rs @@ -1,5 +1,6 @@ extern crate alloc; use alloc::collections::BTreeSet; +use core::str::FromStr; use cid::Cid; use codec::Encode; @@ -8,12 +9,12 @@ use frame_support::{ traits::Hooks, PalletId, }; use frame_system::pallet_prelude::BlockNumberFor; -use multihash_codetable::{Code, MultihashDigest}; use pallet_market::{BalanceOf, ClientDealProposal, DealProposal, DealState}; use primitives_proofs::{ DealId, RegisteredPoStProof, RegisteredSealProof, SectorId, SectorNumber, MAX_DEALS_PER_SECTOR, MAX_TERMINATIONS_PER_CALL, }; +use primitives_shared::commitment::{Commitment, CommitmentKind}; use sp_core::{bounded_vec, Pair}; use sp_runtime::{ traits::{IdentifyAccount, IdentityLookup, Verify}, @@ -25,7 +26,7 @@ use crate::{ fault::{ DeclareFaultsParams, DeclareFaultsRecoveredParams, FaultDeclaration, RecoveryDeclaration, }, - pallet::{CID_CODEC, DECLARATIONS_MAX}, + pallet::DECLARATIONS_MAX, partition::{PartitionNumber, MAX_PARTITIONS_PER_DEADLINE}, proofs::{PoStProof, SubmitWindowedPoStParams}, sector::SectorPreCommitInfo, @@ -178,10 +179,6 @@ fn events() -> Vec { evt } -fn cid_of(data: &str) -> cid::Cid { - Cid::new_v1(CID_CODEC, Code::Blake2b256.digest(data.as_bytes())) -} - fn sign(pair: &sp_core::sr25519::Pair, bytes: &[u8]) -> MultiSignature { MultiSignature::Sr25519(pair.sign(bytes)) } @@ -293,20 +290,28 @@ struct SectorPreCommitInfoBuilder { impl Default for SectorPreCommitInfoBuilder { fn default() -> Self { + let unsealed_cid = + Cid::from_str("baga6ea4seaqgi5lnnv4wi5lnnv4wi5lnnv4wi5lnnv4wi5lnnv4wi5lnnv4wi5i") + .unwrap() + .to_bytes() + .try_into() + .expect("hash is always 32 bytes"); + + // TODO: This cid is not correct and it's only used as a place holder for the correct one + let sealed_cid = + Cid::from_str("baga6ea4seaqgi5lnnv4wi5lnnv4wi5lnnv4wi5lnnv4wi5lnnv4wi5lnnv4wi5i") + .unwrap() + .to_bytes() + .try_into() + .expect("hash is always 32 bytes"); + Self { seal_proof: RegisteredSealProof::StackedDRG2KiBV1P1, sector_number: 1, - sealed_cid: cid_of("sealed_cid") - .to_bytes() - .try_into() - .expect("hash is always 32 bytes"), + sealed_cid, deal_ids: bounded_vec![0, 1], expiration: 120 * MINUTES, - // TODO(@th7nder,#92,19/07/2024): compute_commd not yet implemented. - unsealed_cid: cid_of("placeholder-to-be-done") - .to_bytes() - .try_into() - .expect("hash is always 32 bytes"), + unsealed_cid, } } } @@ -327,8 +332,9 @@ impl SectorPreCommitInfoBuilder { self } - pub fn unsealed_cid(mut self, unsealed_cid: SectorId) -> Self { - self.unsealed_cid = unsealed_cid; + pub fn unsealed_cid(mut self, unsealed_cid: &str) -> Self { + let cid = Cid::from_str(unsealed_cid).expect("valid unsealed_cid"); + self.unsealed_cid = BoundedVec::try_from(cid.to_bytes()).unwrap(); self } @@ -361,12 +367,16 @@ struct DealProposalBuilder { impl Default for DealProposalBuilder { fn default() -> Self { + let piece_commitment = + Commitment::new(*b"dummydummydummydummydummydummydu", CommitmentKind::Piece); + Self { - piece_cid: cid_of("polka-storage-data") + piece_cid: piece_commitment + .cid() .to_bytes() .try_into() .expect("hash is always 32 bytes"), - piece_size: 18, + piece_size: 128, // Smallest piece size available for sector client: account(BOB), provider: account(ALICE), label: bounded_vec![0xb, 0xe, 0xe, 0xf], diff --git a/pallets/storage-provider/src/tests/pre_commit_sectors.rs b/pallets/storage-provider/src/tests/pre_commit_sectors.rs index 9b6abec3e..aefc679a1 100644 --- a/pallets/storage-provider/src/tests/pre_commit_sectors.rs +++ b/pallets/storage-provider/src/tests/pre_commit_sectors.rs @@ -9,7 +9,7 @@ use crate::{ pallet::{Error, Event, StorageProviders}, sector::{SectorPreCommitInfo, MAX_SECTORS}, tests::{ - account, cid_of, events, publish_deals, register_storage_provider, run_to_block, Balances, + account, events, publish_deals, register_storage_provider, run_to_block, Balances, MaxProveCommitDuration, MaxSectorExpiration, RuntimeEvent, RuntimeOrigin, SectorPreCommitInfoBuilder, StorageProvider, Test, ALICE, CHARLIE, INITIAL_FUNDS, }, @@ -25,7 +25,9 @@ fn successfully_precommited() { publish_deals(storage_provider); // Sector to be pre-committed. - let sector = SectorPreCommitInfoBuilder::default().build(); + let sector = SectorPreCommitInfoBuilder::default() + .unsealed_cid("baga6ea4seaqeqgpphr6lmjhddjprb2etcfiml4sgr2kpju7kscfdj7227itm4hq") + .build(); // Check starting balance assert_eq!( @@ -78,12 +80,6 @@ fn successfully_precommited_no_deals() { let sector = SectorPreCommitInfoBuilder::default() // No sectors -> No CommD verification .deals(bounded_vec![]) - .unsealed_cid( - cid_of("cc-unsealed-cid") - .to_bytes() - .try_into() - .expect("hash is always 32 bytes"), - ) .build(); // Run pre commit extrinsic @@ -141,6 +137,9 @@ fn successfully_precommited_batch() { .try_push( SectorPreCommitInfoBuilder::default() .sector_number(sector_number) + .unsealed_cid( + "baga6ea4seaqeqgpphr6lmjhddjprb2etcfiml4sgr2kpju7kscfdj7227itm4hq", + ) .build(), ) .expect("BoundedVec should fit all 6 elements"); @@ -223,7 +222,9 @@ fn fails_sector_number_already_used() { publish_deals(storage_provider); // Sector to be pre-committed - let sector = SectorPreCommitInfoBuilder::default().build(); + let sector = SectorPreCommitInfoBuilder::default() + .unsealed_cid("baga6ea4seaqeqgpphr6lmjhddjprb2etcfiml4sgr2kpju7kscfdj7227itm4hq") + .build(); // Run pre commit extrinsic assert_ok!(StorageProvider::pre_commit_sectors( @@ -251,12 +252,8 @@ fn fails_declared_commd_not_matching() { // Sector to be pre-committed let sector = SectorPreCommitInfoBuilder::default() - .unsealed_cid( - cid_of("different-unsealed-cid") - .to_bytes() - .try_into() - .expect("hash is always 32 bytes"), - ) + // wrong cid for for the sector + .unsealed_cid("baga6ea4seaqgi5lnnv4wi5lnnv4wi5lnnv4wi5lnnv4wi5lnnv4wi5lnnv4wi5i") .build(); assert_noop!( diff --git a/pallets/storage-provider/src/tests/prove_commit_sectors.rs b/pallets/storage-provider/src/tests/prove_commit_sectors.rs index 2fafb9d44..026e13e13 100644 --- a/pallets/storage-provider/src/tests/prove_commit_sectors.rs +++ b/pallets/storage-provider/src/tests/prove_commit_sectors.rs @@ -32,6 +32,7 @@ fn successfully_prove_sector() { // Sector data let sector = SectorPreCommitInfoBuilder::default() .sector_number(sector_number) + .unsealed_cid("baga6ea4seaqeqgpphr6lmjhddjprb2etcfiml4sgr2kpju7kscfdj7227itm4hq") .build(); // Run pre commit extrinsic @@ -116,6 +117,9 @@ fn successfully_prove_multiple_sectors() { .try_push( SectorPreCommitInfoBuilder::default() .sector_number(sector_number) + .unsealed_cid( + "baga6ea4seaqeqgpphr6lmjhddjprb2etcfiml4sgr2kpju7kscfdj7227itm4hq", + ) .build(), ) .expect("BoundedVec should fit all 6 elements"); @@ -268,6 +272,7 @@ fn fails_prove_commit_after_deadline() { // Sector data let sector = SectorPreCommitInfoBuilder::default() .sector_number(sector_number) + .unsealed_cid("baga6ea4seaqeqgpphr6lmjhddjprb2etcfiml4sgr2kpju7kscfdj7227itm4hq") .build(); // Run pre commit extrinsic diff --git a/primitives/shared/src/piece.rs b/primitives/shared/src/piece.rs index 79b3e3508..1753328bf 100644 --- a/primitives/shared/src/piece.rs +++ b/primitives/shared/src/piece.rs @@ -60,6 +60,9 @@ impl Add for UnpaddedPieceSize { pub struct PaddedPieceSize(u64); impl PaddedPieceSize { + /// The minimum pice size + pub const MIN: PaddedPieceSize = PaddedPieceSize(128); + /// Initialize new padded piece size. Error is returned if the size is /// invalid. pub fn new(size: u64) -> Result { From a10f0d4bdda92740688a44d994bbda281b5341ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20=C4=8Cerni=C4=8D?= Date: Tue, 8 Oct 2024 15:04:07 +0200 Subject: [PATCH 17/24] rename crate --- Cargo.lock | 22 +++++++++---------- Cargo.toml | 4 ++-- cli/polka-storage-provider/Cargo.toml | 2 +- .../src/commands/utils.rs | 2 +- cli/polka-storage-provider/src/commp.rs | 7 +----- cli/polka-storage-provider/src/storage.rs | 2 +- pallets/market/Cargo.toml | 2 +- pallets/market/src/commd.rs | 7 +++--- pallets/market/src/lib.rs | 5 +---- pallets/market/src/test.rs | 3 +-- pallets/storage-provider/Cargo.toml | 2 +- pallets/storage-provider/src/lib.rs | 2 +- pallets/storage-provider/src/tests/mod.rs | 2 +- primitives/{shared => commitment}/Cargo.toml | 4 +--- .../mod.rs => commitment/src/lib.rs} | 11 ++++++++-- .../{shared => commitment}/src/piece.rs | 0 .../src/commitment => commitment/src}/zero.rs | 4 +++- primitives/shared/src/lib.rs | 8 ------- 18 files changed, 39 insertions(+), 50 deletions(-) rename primitives/{shared => commitment}/Cargo.toml (88%) rename primitives/{shared/src/commitment/mod.rs => commitment/src/lib.rs} (95%) rename primitives/{shared => commitment}/src/piece.rs (100%) rename primitives/{shared/src/commitment => commitment/src}/zero.rs (97%) delete mode 100644 primitives/shared/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index fca0f13df..d0f79b676 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8223,8 +8223,8 @@ dependencies = [ "pallet-balances", "pallet-storage-provider", "parity-scale-codec", + "primitives-commitment", "primitives-proofs", - "primitives-shared", "scale-info", "sha2 0.10.8", "sp-arithmetic 26.0.0", @@ -8700,8 +8700,8 @@ dependencies = [ "pallet-balances", "pallet-market", "parity-scale-codec", + "primitives-commitment", "primitives-proofs", - "primitives-shared", "rstest", "scale-info", "sp-arithmetic 26.0.0", @@ -9410,8 +9410,8 @@ dependencies = [ "mater", "parity-scale-codec", "polka-storage-proofs", + "primitives-commitment", "primitives-proofs", - "primitives-shared", "rand", "rocksdb", "sc-cli", @@ -10862,6 +10862,14 @@ dependencies = [ "uint", ] +[[package]] +name = "primitives-commitment" +version = "0.1.0" +dependencies = [ + "cid 0.11.1", + "rand", +] + [[package]] name = "primitives-proofs" version = "0.1.0" @@ -10877,14 +10885,6 @@ dependencies = [ "sp-runtime 39.0.0", ] -[[package]] -name = "primitives-shared" -version = "0.1.0" -dependencies = [ - "cid 0.11.1", - "rand", -] - [[package]] name = "prioritized-metered-channel" version = "0.6.1" diff --git a/Cargo.toml b/Cargo.toml index 0c95e1d0b..e0be573f1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,8 +16,8 @@ members = [ "pallets/market", "pallets/proofs", "pallets/storage-provider", + "primitives/commitment", "primitives/proofs", - "primitives/shared", "runtime", "storage/mater", "storage/mater-cli", @@ -127,8 +127,8 @@ pallet-proofs = { path = "pallets/proofs", default-features = false } pallet-storage-provider = { path = "pallets/storage-provider", default-features = false } polka-storage-proofs = { path = "lib/polka-storage-proofs", default-features = false } polka-storage-runtime = { path = "runtime" } +primitives-commitment = { path = "primitives/commitment" } primitives-proofs = { path = "primitives/proofs", default-features = false } -primitives-shared = { path = "primitives/shared" } storagext = { path = "cli/polka-storage/storagext" } # FileCoin proofs diff --git a/cli/polka-storage-provider/Cargo.toml b/cli/polka-storage-provider/Cargo.toml index 7a6e2245c..2b77e8c77 100644 --- a/cli/polka-storage-provider/Cargo.toml +++ b/cli/polka-storage-provider/Cargo.toml @@ -25,8 +25,8 @@ futures = { workspace = true } hex = { workspace = true } jsonrpsee = { workspace = true, features = ["http-client", "macros", "server", "ws-client"] } polka-storage-proofs = { workspace = true, features = ["std", "substrate"] } +primitives-commitment = { workspace = true } primitives-proofs = { workspace = true, features = ["std"] } -primitives-shared = { workspace = true } rand = { workspace = true } rocksdb = { workspace = true } sc-cli = { workspace = true } diff --git a/cli/polka-storage-provider/src/commands/utils.rs b/cli/polka-storage-provider/src/commands/utils.rs index 9b82071ce..d16a0c8fd 100644 --- a/cli/polka-storage-provider/src/commands/utils.rs +++ b/cli/polka-storage-provider/src/commands/utils.rs @@ -10,8 +10,8 @@ use polka_storage_proofs::{ porep::{self, sealer::Sealer}, types::PieceInfo, }; +use primitives_commitment::piece::PaddedPieceSize; use primitives_proofs::RegisteredSealProof; -use primitives_shared::piece::PaddedPieceSize; use crate::{ commp::{calculate_piece_commitment, CommPError, ZeroPaddingReader}, diff --git a/cli/polka-storage-provider/src/commp.rs b/cli/polka-storage-provider/src/commp.rs index 68d0047c7..d66579eea 100644 --- a/cli/polka-storage-provider/src/commp.rs +++ b/cli/polka-storage-provider/src/commp.rs @@ -5,11 +5,7 @@ use filecoin_hashers::{ Domain, }; use fr32::Fr32Reader; -use primitives_shared::{ - commitment::{Commitment, CommitmentKind}, - piece::PaddedPieceSize, - NODE_SIZE, -}; +use primitives_commitment::{NODE_SIZE, piece::PaddedPieceSize, Commitment, CommitmentKind}; use storage_proofs_core::merkle::BinaryMerkleTree; use thiserror::Error; @@ -106,7 +102,6 @@ mod tests { use std::io::{Cursor, Read}; use primitives_proofs::SectorSize; - use primitives_shared::piece::PaddedPieceSize; use super::{calculate_piece_commitment, ZeroPaddingReader}; diff --git a/cli/polka-storage-provider/src/storage.rs b/cli/polka-storage-provider/src/storage.rs index 82b94c908..f317beb2e 100644 --- a/cli/polka-storage-provider/src/storage.rs +++ b/cli/polka-storage-provider/src/storage.rs @@ -10,7 +10,7 @@ use axum::{ }; use futures::{TryFutureExt, TryStreamExt}; use mater::Cid; -use primitives_shared::piece::PaddedPieceSize; +use primitives_commitment::piece::PaddedPieceSize; use tokio::{ fs::{self, File}, io::{AsyncRead, BufWriter}, diff --git a/pallets/market/Cargo.toml b/pallets/market/Cargo.toml index 495550fb4..900c3a635 100644 --- a/pallets/market/Cargo.toml +++ b/pallets/market/Cargo.toml @@ -21,8 +21,8 @@ codec = { workspace = true, default-features = false, features = ["derive"] } hex = { workspace = true, default-features = false, features = ["alloc"] } log = { workspace = true } multihash-codetable = { workspace = true, features = ["blake2b"] } +primitives-commitment = { workspace = true } primitives-proofs = { workspace = true, default-features = false } -primitives-shared = { workspace = true } scale-info = { workspace = true, default-features = false, features = ["derive"] } sha2 = { workspace = true } diff --git a/pallets/market/src/commd.rs b/pallets/market/src/commd.rs index 8bcf9727a..9fea254ae 100644 --- a/pallets/market/src/commd.rs +++ b/pallets/market/src/commd.rs @@ -1,12 +1,11 @@ extern crate alloc; use alloc::vec::Vec; -use primitives_proofs::SectorSize; -use primitives_shared::{ - commitment::{zero_piece_commitment, Commitment, CommitmentKind}, +use primitives_commitment::{ piece::{PaddedPieceSize, UnpaddedPieceSize}, - NODE_SIZE, + zero_piece_commitment, Commitment, CommitmentKind, NODE_SIZE, }; +use primitives_proofs::SectorSize; use sha2::{Digest, Sha256}; // Ensure that the pieces are correct sizes diff --git a/pallets/market/src/lib.rs b/pallets/market/src/lib.rs index 74d9f964f..54a0f26bd 100644 --- a/pallets/market/src/lib.rs +++ b/pallets/market/src/lib.rs @@ -38,15 +38,12 @@ pub mod pallet { PalletId, }; use frame_system::{pallet_prelude::*, Config as SystemConfig, Pallet as System}; + use primitives_commitment::{piece::PaddedPieceSize, Commitment, CommitmentKind}; use primitives_proofs::{ ActiveDeal, ActiveSector, DealId, Market, RegisteredSealProof, SectorDeal, SectorId, SectorNumber, SectorSize, StorageProviderValidation, MAX_DEALS_FOR_ALL_SECTORS, MAX_DEALS_PER_SECTOR, MAX_SECTORS_PER_CALL, }; - use primitives_shared::{ - commitment::{Commitment, CommitmentKind}, - piece::PaddedPieceSize, - }; use scale_info::TypeInfo; use sp_arithmetic::traits::BaseArithmetic; use sp_std::vec::Vec; diff --git a/pallets/market/src/test.rs b/pallets/market/src/test.rs index 5eeffd80a..8ce15affb 100644 --- a/pallets/market/src/test.rs +++ b/pallets/market/src/test.rs @@ -12,7 +12,6 @@ use primitives_proofs::{ ActiveDeal, ActiveSector, DealId, Market as MarketTrait, RegisteredSealProof, SectorDeal, MAX_DEALS_PER_SECTOR, }; -use primitives_shared::commitment::Commitment; use sp_core::H256; use sp_runtime::AccountId32; @@ -1775,7 +1774,7 @@ impl> Default for DealProposalB fn default() -> Self { let piece_commitment = Commitment::new( *b"dummydummydummydummydummydummydu", - primitives_shared::commitment::CommitmentKind::Piece, + CommitmentKind::Piece, ); Self { diff --git a/pallets/storage-provider/Cargo.toml b/pallets/storage-provider/Cargo.toml index 2631c6447..e75fd3815 100644 --- a/pallets/storage-provider/Cargo.toml +++ b/pallets/storage-provider/Cargo.toml @@ -18,8 +18,8 @@ targets = ["x86_64-unknown-linux-gnu"] cid = { workspace = true, features = ["alloc"] } codec = { workspace = true, default-features = false, features = ["derive"] } log = { workspace = true, features = ["kv"] } +primitives-commitment = { workspace = true } primitives-proofs = { workspace = true, default-features = false } -primitives-shared = { workspace = true } scale-info = { workspace = true, default-features = false, features = ["derive"] } sp-arithmetic = { workspace = true, default-features = false } sp-core = { workspace = true, default-features = false } diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index e12851c13..115f40487 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -55,11 +55,11 @@ pub mod pallet { pallet_prelude::{BlockNumberFor, *}, Config as SystemConfig, }; + use primitives_commitment::{Commitment, CommitmentKind}; use primitives_proofs::{ Market, RegisteredPoStProof, RegisteredSealProof, SectorNumber, StorageProviderValidation, MAX_SECTORS_PER_CALL, }; - use primitives_shared::commitment::{Commitment, CommitmentKind}; use scale_info::TypeInfo; use sp_arithmetic::traits::Zero; diff --git a/pallets/storage-provider/src/tests/mod.rs b/pallets/storage-provider/src/tests/mod.rs index 369010f0d..9dc3a810f 100644 --- a/pallets/storage-provider/src/tests/mod.rs +++ b/pallets/storage-provider/src/tests/mod.rs @@ -10,11 +10,11 @@ use frame_support::{ }; use frame_system::pallet_prelude::BlockNumberFor; use pallet_market::{BalanceOf, ClientDealProposal, DealProposal, DealState}; +use primitives_commitment::{Commitment, CommitmentKind}; use primitives_proofs::{ DealId, RegisteredPoStProof, RegisteredSealProof, SectorId, SectorNumber, MAX_DEALS_PER_SECTOR, MAX_TERMINATIONS_PER_CALL, }; -use primitives_shared::commitment::{Commitment, CommitmentKind}; use sp_core::{bounded_vec, Pair}; use sp_runtime::{ traits::{IdentifyAccount, IdentityLookup, Verify}, diff --git a/primitives/shared/Cargo.toml b/primitives/commitment/Cargo.toml similarity index 88% rename from primitives/shared/Cargo.toml rename to primitives/commitment/Cargo.toml index 16ad9b287..a3f84ee4d 100644 --- a/primitives/shared/Cargo.toml +++ b/primitives/commitment/Cargo.toml @@ -3,7 +3,7 @@ authors.workspace = true edition.workspace = true homepage.workspace = true license-file.workspace = true -name = "primitives-shared" +name = "primitives-commitment" repository.workspace = true version = "0.1.0" @@ -15,5 +15,3 @@ rand = { workspace = true, features = ["std", "std_rng"] } [lints] workspace = true - -[features] diff --git a/primitives/shared/src/commitment/mod.rs b/primitives/commitment/src/lib.rs similarity index 95% rename from primitives/shared/src/commitment/mod.rs rename to primitives/commitment/src/lib.rs index f0c2cc092..153d34afd 100644 --- a/primitives/shared/src/commitment/mod.rs +++ b/primitives/commitment/src/lib.rs @@ -1,9 +1,15 @@ +#![no_std] + +pub mod piece; mod zero; use cid::{multihash::Multihash, Cid}; use crate::piece::PaddedPieceSize; +/// Merkle tree node size in bytes. +pub const NODE_SIZE: usize = 32; + /// Filecoin piece or sector data commitment merkle node/root (CommP & CommD) /// /// https://github.com/multiformats/multicodec/blob/badcfe56bb7e0bbb06b60d57565186cd6be1f932/table.csv#L554 @@ -37,6 +43,7 @@ pub enum CommitmentKind { } impl CommitmentKind { + /// Returns the [Multicodec](https://github.com/multiformats/multicodec/blob/master/table.csv) code for the commitment kind. fn multicodec(&self) -> u64 { match self { CommitmentKind::Piece | CommitmentKind::Data => FIL_COMMITMENT_UNSEALED, @@ -44,6 +51,7 @@ impl CommitmentKind { } } + /// Returns the [Multihash](https://github.com/multiformats/multicodec/blob/master/table.csv) code for the commitment kind. fn multihash(&self) -> u64 { match self { CommitmentKind::Piece | CommitmentKind::Data => SHA2_256_TRUNC254_PADDED, @@ -121,8 +129,7 @@ pub fn zero_piece_commitment(size: PaddedPieceSize) -> Commitment { mod tests { use cid::{multihash::Multihash, Cid}; - // use rand::thread_rng; - use crate::commitment::{ + use crate::{ Commitment, CommitmentKind, FIL_COMMITMENT_SEALED, FIL_COMMITMENT_UNSEALED, POSEIDON_BLS12_381_A1_FC1, SHA2_256_TRUNC254_PADDED, }; diff --git a/primitives/shared/src/piece.rs b/primitives/commitment/src/piece.rs similarity index 100% rename from primitives/shared/src/piece.rs rename to primitives/commitment/src/piece.rs diff --git a/primitives/shared/src/commitment/zero.rs b/primitives/commitment/src/zero.rs similarity index 97% rename from primitives/shared/src/commitment/zero.rs rename to primitives/commitment/src/zero.rs index fa919d95a..f145c06ec 100644 --- a/primitives/shared/src/commitment/zero.rs +++ b/primitives/commitment/src/zero.rs @@ -3,7 +3,9 @@ use crate::piece::PaddedPieceSize; /// Can't generate for: 1, 2, 4, 8, 16, 32, 64 bytes const SKIP: u32 = 7; -/// Returns a zero piece commitment for a specified piece size +/// Returns a zero piece commitment for a specified piece size. Zero piece +/// commitment is a calculated piece commitment for a 0-filled piece. This +/// commitment is usually used when adding pieces to a sector as a padding. pub fn zero_piece_commitment(size: PaddedPieceSize) -> [u8; 32] { let level = size.trailing_zeros() - SKIP; PIECE_COMMS[level as usize] diff --git a/primitives/shared/src/lib.rs b/primitives/shared/src/lib.rs deleted file mode 100644 index 75acb0126..000000000 --- a/primitives/shared/src/lib.rs +++ /dev/null @@ -1,8 +0,0 @@ -#![no_std] - -pub mod commitment; -pub mod piece; - -/// Merkle tree node size in bytes. -/// TODO: Where should this be moved to? -pub const NODE_SIZE: usize = 32; From f14c8fccf775e262db74070ce4ca132a2a782c53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20=C4=8Cerni=C4=8D?= Date: Tue, 8 Oct 2024 15:08:22 +0200 Subject: [PATCH 18/24] use hashed directly --- pallets/market/src/commd.rs | 8 +++----- pallets/market/src/test.rs | 1 + 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/pallets/market/src/commd.rs b/pallets/market/src/commd.rs index 9fea254ae..7f147173f 100644 --- a/pallets/market/src/commd.rs +++ b/pallets/market/src/commd.rs @@ -197,14 +197,12 @@ pub fn piece_hash(a: &[u8], b: &[u8]) -> [u8; 32] { buf[..NODE_SIZE].copy_from_slice(a); buf[NODE_SIZE..].copy_from_slice(b); - let hashed = Sha256::digest(buf); - let mut result = [0u8; 32]; - result.copy_from_slice(&hashed); + let mut hashed = Sha256::digest(buf); // strip last two bits, to ensure result is in Fr. - result[31] &= 0b0011_1111; + hashed[31] &= 0b0011_1111; - result + hashed.into() } #[derive(Debug)] diff --git a/pallets/market/src/test.rs b/pallets/market/src/test.rs index 8ce15affb..ba4391b17 100644 --- a/pallets/market/src/test.rs +++ b/pallets/market/src/test.rs @@ -12,6 +12,7 @@ use primitives_proofs::{ ActiveDeal, ActiveSector, DealId, Market as MarketTrait, RegisteredSealProof, SectorDeal, MAX_DEALS_PER_SECTOR, }; +use primitives_commitment::{Commitment, CommitmentKind}; use sp_core::H256; use sp_runtime::AccountId32; From b6c89350dfbc647758765952c3467411d62849b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20=C4=8Cerni=C4=8D?= Date: Tue, 8 Oct 2024 15:26:43 +0200 Subject: [PATCH 19/24] fi --- cli/polka-storage-provider/src/commp.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/cli/polka-storage-provider/src/commp.rs b/cli/polka-storage-provider/src/commp.rs index d66579eea..3b8ea1dda 100644 --- a/cli/polka-storage-provider/src/commp.rs +++ b/cli/polka-storage-provider/src/commp.rs @@ -103,6 +103,7 @@ mod tests { use primitives_proofs::SectorSize; + use primitives_commitment::piece::PaddedPieceSize; use super::{calculate_piece_commitment, ZeroPaddingReader}; #[test] From f201214b6e3460028944c55de009c6a01865d5f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20=C4=8Cerni=C4=8D?= Date: Tue, 8 Oct 2024 15:41:34 +0200 Subject: [PATCH 20/24] add updated scale --- cli/artifacts/metadata.scale | Bin 148990 -> 149059 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/cli/artifacts/metadata.scale b/cli/artifacts/metadata.scale index 88a8c0484e243142cea42d8895d9026cf8b170ba..5194aa60fd1c1109fbe4d191273fa9faa1ba1a0c 100644 GIT binary patch delta 125 zcmey@&3U+obHln4QxTDPBkxQn1_lO^cne=3gONcd-qg?t#AITSh&MKJ2eO!fEHe`j zi-m!Km1*)ve}m@NCEH(@Fix>$G}+#;iSZSytcY`dZmtUh4~q|&p-_~XlUS0PqL5lt SlwUM`|1?IW?L9jfi+Ta0u_j&s delta 47 zcmV+~0MP%#j0ygW39zno2?P{hGD~Sf004vEbhqDh0mwZ8Ft@0(0qO;pf`b7fx6!x( Fa*fIo6MO&w From fcfe28e6db98268156621abb24199b1777e624a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20=C4=8Cerni=C4=8D?= Date: Wed, 9 Oct 2024 09:34:50 +0200 Subject: [PATCH 21/24] fix real world test --- maat/tests/real_world.rs | 18 ++++++++++++------ pallets/market/src/lib.rs | 7 ++++++- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/maat/tests/real_world.rs b/maat/tests/real_world.rs index 85bfcd1ec..f0275bcf1 100644 --- a/maat/tests/real_world.rs +++ b/maat/tests/real_world.rs @@ -141,10 +141,10 @@ async fn publish_storage_deals( // Publish a storage deal let husky_storage_deal = DealProposal { piece_cid: cid::Cid::try_from( - "bafybeihxgc67fwhdoxo2klvmsetswdmwwz3brpwwl76qizbsl6ypro6vxq", + "baga6ea4seaqgi5lnnv4wi5lnnv4wi5lnnv4wi5lnnv4wi5lnnv4wi5lnnv4wi5i", ) .expect("valid CID"), - piece_size: 1278, + piece_size: 2048, client: alice.account_id().clone(), provider: charlie.account_id().clone(), label: "My lovely Husky (husky.jpg)".to_owned(), @@ -177,18 +177,24 @@ async fn pre_commit_sectors(client: &storagext::Client, charlie: &Keypa where Keypair: subxt::tx::Signer, { - // Pre commit sectors + // Unsealed sector commitment + let unsealed_cid = + cid::Cid::try_from("baga6ea4seaqgi5lnnv4wi5lnnv4wi5lnnv4wi5lnnv4wi5lnnv4wi5lnnv4wi5i") + .expect("valid CID"); + + // Sealed sector commitment. + // Currently a placeholder value. let placeholder_cid = cid::Cid::try_from("bafk2bzaceajreoxfdcpdvitpvxm7vkpvcimlob5ejebqgqidjkz4qoug4q6zu") - .unwrap(); + .expect("valid CID"); let sectors_pre_commit_info = vec![SectorPreCommitInfo { seal_proof: primitives_proofs::RegisteredSealProof::StackedDRG2KiBV1P1, sector_number: 1, - sealed_cid: placeholder_cid.clone(), + sealed_cid: placeholder_cid, deal_ids: vec![0], expiration: 165, - unsealed_cid: placeholder_cid, + unsealed_cid, }]; let result = client diff --git a/pallets/market/src/lib.rs b/pallets/market/src/lib.rs index 54a0f26bd..f96f360ac 100644 --- a/pallets/market/src/lib.rs +++ b/pallets/market/src/lib.rs @@ -525,6 +525,8 @@ pub mod pallet { DealDurationOutOfBounds, /// Deal's piece_cid is invalid. InvalidPieceCid(cid::Error), + /// Deal's piece_size is invalid. + InvalidPieceSize(&'static str), /// CommD related error CommD(&'static str), } @@ -553,6 +555,9 @@ pub mod pallet { ProposalError::InvalidPieceCid(_err) => { write!(f, "ProposalError::InvalidPieceCid") } + ProposalError::InvalidPieceSize(err) => { + write!(f, "ProposalError::InvalidPieceSize: {}", err) + } ProposalError::CommD(err) => { write!(f, "ProposalError::CommD: {}", err) } @@ -974,7 +979,7 @@ pub mod pallet { let commitment = Commitment::from_cid(&cid, CommitmentKind::Piece) .map_err(|err| ProposalError::CommD(err))?; let size = PaddedPieceSize::new(p.piece_size) - .map_err(|err| ProposalError::CommD(err))?; + .map_err(|err| ProposalError::InvalidPieceSize(err))?; Ok(crate::commd::PieceInfo { size, commitment }) }) From 04b9744ee9382fe88f3de422d091692f2b1a08af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20=C4=8Cerni=C4=8D?= Date: Wed, 9 Oct 2024 10:44:43 +0200 Subject: [PATCH 22/24] fix fmt --- cli/polka-storage-provider/src/commp.rs | 4 ++-- pallets/market/src/test.rs | 8 +++----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/cli/polka-storage-provider/src/commp.rs b/cli/polka-storage-provider/src/commp.rs index 3b8ea1dda..696998073 100644 --- a/cli/polka-storage-provider/src/commp.rs +++ b/cli/polka-storage-provider/src/commp.rs @@ -5,7 +5,7 @@ use filecoin_hashers::{ Domain, }; use fr32::Fr32Reader; -use primitives_commitment::{NODE_SIZE, piece::PaddedPieceSize, Commitment, CommitmentKind}; +use primitives_commitment::{piece::PaddedPieceSize, Commitment, CommitmentKind, NODE_SIZE}; use storage_proofs_core::merkle::BinaryMerkleTree; use thiserror::Error; @@ -101,9 +101,9 @@ pub enum CommPError { mod tests { use std::io::{Cursor, Read}; + use primitives_commitment::piece::PaddedPieceSize; use primitives_proofs::SectorSize; - use primitives_commitment::piece::PaddedPieceSize; use super::{calculate_piece_commitment, ZeroPaddingReader}; #[test] diff --git a/pallets/market/src/test.rs b/pallets/market/src/test.rs index ba4391b17..8b842fac9 100644 --- a/pallets/market/src/test.rs +++ b/pallets/market/src/test.rs @@ -8,11 +8,11 @@ use frame_support::{ traits::Currency, BoundedVec, }; +use primitives_commitment::{Commitment, CommitmentKind}; use primitives_proofs::{ ActiveDeal, ActiveSector, DealId, Market as MarketTrait, RegisteredSealProof, SectorDeal, MAX_DEALS_PER_SECTOR, }; -use primitives_commitment::{Commitment, CommitmentKind}; use sp_core::H256; use sp_runtime::AccountId32; @@ -1773,10 +1773,8 @@ pub struct DealProposalBuilder { impl> Default for DealProposalBuilder { fn default() -> Self { - let piece_commitment = Commitment::new( - *b"dummydummydummydummydummydummydu", - CommitmentKind::Piece, - ); + let piece_commitment = + Commitment::new(*b"dummydummydummydummydummydummydu", CommitmentKind::Piece); Self { piece_cid: piece_commitment From fb93b43ec1c0dc9f3b9edc62586a9649486a1cfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20=C4=8Cerni=C4=8D?= Date: Wed, 9 Oct 2024 13:33:02 +0200 Subject: [PATCH 23/24] pr review related changes --- Cargo.lock | 3 +- pallets/market/Cargo.toml | 1 - pallets/market/src/lib.rs | 15 ++++--- pallets/market/src/mock.rs | 6 +-- primitives/commitment/Cargo.toml | 2 + .../commitment}/src/commd.rs | 40 +++++++++---------- primitives/commitment/src/lib.rs | 1 + primitives/commitment/src/piece.rs | 11 ++++- 8 files changed, 45 insertions(+), 34 deletions(-) rename {pallets/market => primitives/commitment}/src/commd.rs (92%) diff --git a/Cargo.lock b/Cargo.lock index d0f79b676..ae6321148 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8226,7 +8226,6 @@ dependencies = [ "primitives-commitment", "primitives-proofs", "scale-info", - "sha2 0.10.8", "sp-arithmetic 26.0.0", "sp-core 34.0.0", "sp-io 38.0.0", @@ -10867,7 +10866,9 @@ name = "primitives-commitment" version = "0.1.0" dependencies = [ "cid 0.11.1", + "primitives-proofs", "rand", + "sha2 0.10.8", ] [[package]] diff --git a/pallets/market/Cargo.toml b/pallets/market/Cargo.toml index 900c3a635..59abe0ecd 100644 --- a/pallets/market/Cargo.toml +++ b/pallets/market/Cargo.toml @@ -24,7 +24,6 @@ multihash-codetable = { workspace = true, features = ["blake2b"] } primitives-commitment = { workspace = true } primitives-proofs = { workspace = true, default-features = false } scale-info = { workspace = true, default-features = false, features = ["derive"] } -sha2 = { workspace = true } # frame deps frame-benchmarking = { workspace = true, default-features = false, optional = true } diff --git a/pallets/market/src/lib.rs b/pallets/market/src/lib.rs index f96f360ac..784266a48 100644 --- a/pallets/market/src/lib.rs +++ b/pallets/market/src/lib.rs @@ -7,8 +7,6 @@ #![cfg_attr(not(feature = "std"), no_std)] -mod commd; - pub use pallet::*; #[cfg(test)] @@ -38,7 +36,11 @@ pub mod pallet { PalletId, }; use frame_system::{pallet_prelude::*, Config as SystemConfig, Pallet as System}; - use primitives_commitment::{piece::PaddedPieceSize, Commitment, CommitmentKind}; + use primitives_commitment::{ + commd::compute_unsealed_sector_commitment, + piece::{PaddedPieceSize, PieceInfo}, + Commitment, CommitmentKind, + }; use primitives_proofs::{ ActiveDeal, ActiveSector, DealId, Market, RegisteredSealProof, SectorDeal, SectorId, SectorNumber, SectorSize, StorageProviderValidation, MAX_DEALS_FOR_ALL_SECTORS, @@ -48,8 +50,6 @@ pub mod pallet { use sp_arithmetic::traits::BaseArithmetic; use sp_std::vec::Vec; - use crate::commd::compute_unsealed_sector_commitment; - pub const LOG_TARGET: &'static str = "runtime::market"; /// Allows to extract Balance of an account via the Config::Currency associated type. @@ -969,11 +969,10 @@ pub mod pallet { /// fn compute_commd<'a>( - proposals: impl IntoIterator>, + proposals: impl Iterator>, sector_type: RegisteredSealProof, ) -> Result { let pieces = proposals - .into_iter() .map(|p| { let cid = p.cid()?; let commitment = Commitment::from_cid(&cid, CommitmentKind::Piece) @@ -981,7 +980,7 @@ pub mod pallet { let size = PaddedPieceSize::new(p.piece_size) .map_err(|err| ProposalError::InvalidPieceSize(err))?; - Ok(crate::commd::PieceInfo { size, commitment }) + Ok(PieceInfo { size, commitment }) }) .collect::, ProposalError>>(); diff --git a/pallets/market/src/mock.rs b/pallets/market/src/mock.rs index 3b982f5e1..f5dce80d4 100644 --- a/pallets/market/src/mock.rs +++ b/pallets/market/src/mock.rs @@ -115,9 +115,9 @@ pub fn sign(pair: &sp_core::sr25519::Pair, bytes: &[u8]) -> MultiSignature { MultiSignature::Sr25519(pair.sign(bytes)) } -// TODO: Remove this function. The codec and hashing is not correct. This is -// still here because I don't want to make the PR even bigger by changing parts -// of the implementations that are relying on this. +// TODO(#442,@cernicc,09/10/2024): Remove this function. The codec and hashing +// is not correct. This is still here because I don't want to make the PR even +// bigger by changing parts of the implementations that are relying on this. pub fn cid_of(data: &str) -> cid::Cid { let cid_codec = 0x55; Cid::new_v1(cid_codec, Code::Blake2b256.digest(data.as_bytes())) diff --git a/primitives/commitment/Cargo.toml b/primitives/commitment/Cargo.toml index a3f84ee4d..619ee961e 100644 --- a/primitives/commitment/Cargo.toml +++ b/primitives/commitment/Cargo.toml @@ -9,6 +9,8 @@ version = "0.1.0" [dependencies] cid.workspace = true +primitives-proofs.workspace = true +sha2.workspace = true [dev-dependencies] rand = { workspace = true, features = ["std", "std_rng"] } diff --git a/pallets/market/src/commd.rs b/primitives/commitment/src/commd.rs similarity index 92% rename from pallets/market/src/commd.rs rename to primitives/commitment/src/commd.rs index 7f147173f..804048397 100644 --- a/pallets/market/src/commd.rs +++ b/primitives/commitment/src/commd.rs @@ -1,13 +1,15 @@ extern crate alloc; use alloc::vec::Vec; +use core::ops::Deref; -use primitives_commitment::{ - piece::{PaddedPieceSize, UnpaddedPieceSize}, - zero_piece_commitment, Commitment, CommitmentKind, NODE_SIZE, -}; use primitives_proofs::SectorSize; use sha2::{Digest, Sha256}; +use crate::{ + piece::{PaddedPieceSize, PieceInfo, UnpaddedPieceSize}, + zero_piece_commitment, Commitment, CommitmentKind, NODE_SIZE, +}; + // Ensure that the pieces are correct sizes fn ensure_piece_sizes( sector_size: SectorSize, @@ -48,20 +50,12 @@ pub fn compute_unsealed_sector_commitment( // Reduce the pieces to the 1-piece commitment let mut reduction = CommDPieceReduction::new(); - reduction.add_many_pieces(piece_infos); + reduction.add_pieces(piece_infos.iter().copied()); let commitment = reduction.finish().expect("at least one piece was added"); Ok(commitment) } -#[derive(Debug, Clone, Copy)] -pub struct PieceInfo { - /// Piece commitment - pub commitment: Commitment, - /// Piece size - pub size: PaddedPieceSize, -} - /// Reduces the pieces to the data commitment of those pieces. struct CommDPieceReduction { /// Pieces stack @@ -73,9 +67,12 @@ impl CommDPieceReduction { CommDPieceReduction { pieces: Vec::new() } } - // Add many pieces pieces - fn add_many_pieces(&mut self, pieces: &[PieceInfo]) { - pieces.iter().for_each(|piece| self.add_piece(*piece)); + // Add many pieces + fn add_pieces

(&mut self, pieces: P) + where + P: Iterator, + { + pieces.for_each(|p| self.add_piece(p)); } // Add a single piece @@ -89,10 +86,13 @@ impl CommDPieceReduction { // Add padding pieces to the stack until we reduce the current pieces to // the size that is equal to the new piece. With this we achieve that // the new piece will be reduced to a single piece after adding it to - // the stack. - loop { - let last_added_piece_size = self.pieces.last().expect("at least one piece exists").size; - if *last_added_piece_size >= *piece.size { + // the stack. Will always iterate at least once since if it was empty + // the first condition would have triggered and returned. + while let Some(last_piece) = self.pieces.last() { + let last_added_piece_size = last_piece.size; + // We can stop stop adding padding pieces if the last added padding + // piece is the same size as the actual piece. + if last_added_piece_size.deref() >= piece.size.deref() { break; } diff --git a/primitives/commitment/src/lib.rs b/primitives/commitment/src/lib.rs index 153d34afd..b033579fb 100644 --- a/primitives/commitment/src/lib.rs +++ b/primitives/commitment/src/lib.rs @@ -1,5 +1,6 @@ #![no_std] +pub mod commd; pub mod piece; mod zero; diff --git a/primitives/commitment/src/piece.rs b/primitives/commitment/src/piece.rs index 1753328bf..56803a721 100644 --- a/primitives/commitment/src/piece.rs +++ b/primitives/commitment/src/piece.rs @@ -1,6 +1,15 @@ use core::ops::{Add, AddAssign, Deref}; -use crate::NODE_SIZE; +use crate::{Commitment, NODE_SIZE}; + +/// Piece info contains piece commitment and piece size. +#[derive(Debug, Clone, Copy)] +pub struct PieceInfo { + /// Piece commitment + pub commitment: Commitment, + /// Piece size + pub size: PaddedPieceSize, +} /// Size of a piece in bytes. Unpadded piece size should be power of two /// multiple of 127. From cedd928e66ec24ca7fa26461e0a61ca04f8595bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20=C4=8Cerni=C4=8D?= Date: Wed, 9 Oct 2024 14:21:53 +0200 Subject: [PATCH 24/24] some more docs --- primitives/commitment/src/commd.rs | 20 +++++++++++++++++--- primitives/commitment/src/piece.rs | 13 ++++++++++--- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/primitives/commitment/src/commd.rs b/primitives/commitment/src/commd.rs index 804048397..bd1044019 100644 --- a/primitives/commitment/src/commd.rs +++ b/primitives/commitment/src/commd.rs @@ -56,7 +56,20 @@ pub fn compute_unsealed_sector_commitment( Ok(commitment) } -/// Reduces the pieces to the data commitment of those pieces. +/// Reduces pieces passed to their data commitment. The process of the reduction +/// is following: +/// +/// 1. Pieces are added to the stack one by one. +/// 2. After each piece is added, the stack is reduced by combining pieces of +/// the same size. +/// 3. If a piece to be added is larger than the last piece on the stack, +/// padding pieces are added until the last piece on the stack is at least as +/// large as the piece to be added. +/// 4. The process continues until all pieces have been added and reduced. +/// 5. At the end, if there is more than one piece on the stack, padding pieces +/// are added until the stack can be reduced to a single piece. +/// 6. The final single piece represents the data commitment for all the input +/// pieces. struct CommDPieceReduction { /// Pieces stack pieces: Vec, @@ -214,7 +227,8 @@ pub enum CommDError { #[cfg(test)] mod tests { - use std::str::FromStr; + use alloc::string::ToString; + use core::str::FromStr; use cid::Cid; use primitives_proofs::SectorSize; @@ -237,7 +251,7 @@ mod tests { /// Reference: #[test] fn compute_unsealed_sector_cid() { - let pieces = vec![ + let pieces = [ ( Some("baga6ea4seaqknzm22isnhsxt2s4dnw45kfywmhenngqq3nc7jvecakoca6ksyhy"), 256 << 20, diff --git a/primitives/commitment/src/piece.rs b/primitives/commitment/src/piece.rs index 56803a721..4e6a02fb0 100644 --- a/primitives/commitment/src/piece.rs +++ b/primitives/commitment/src/piece.rs @@ -64,7 +64,8 @@ impl Add for UnpaddedPieceSize { } } -/// Size of a piece in bytes with padding. The size should be power of two. +/// Size of a piece in bytes with padding. The size is always a power of two +/// number. #[derive(PartialEq, Debug, Eq, Clone, Copy)] pub struct PaddedPieceSize(u64); @@ -97,8 +98,14 @@ impl PaddedPieceSize { } /// The function accepts arbitrary size and transforms it to the - /// PaddedPieceSize. We first pad the size. After that we take the first - /// power of 2 number. + /// PaddedPieceSize: + /// + /// 1. We first add as many bytes as we get when we add "0" byte after each + /// 127 bytes. That is because we are padding the sector content with + /// "Fr32 padding". + /// 2. We "round" the padded size to the first power of two number. That is + /// needed because we use Binary Merkle Tree for the CommD/CommP + /// computation. pub fn from_arbitrary_size(size: u64) -> Self { let padded_bytes = size + (size / 127); let padded_bytes = padded_bytes.next_power_of_two();