feat(commd): calculate commd

.vscode/settings.json

@@ -1,11 +1,11 @@
 {
-    "files.trimTrailingWhitespace": true,
-    "rust-analyzer.rustfmt.extraArgs": ["+nightly"],
-    "rust-analyzer.checkOnSave": false,
-    "coverage-gutters.coverageBaseDir": "coverage",
-    "coverage-gutters.coverageFileNames": [
-        "pallet-storage-provider/lcov.info",
-        "pallet-market/lcov.info",
-        "mater/lcov.info"
-    ]
+  "files.trimTrailingWhitespace": true,
+  "rust-analyzer.rustfmt.extraArgs": ["+nightly"],
+  "rust-analyzer.checkOnSave": false,
+  "coverage-gutters.coverageBaseDir": "coverage",
+  "coverage-gutters.coverageFileNames": [
+    "pallet-storage-provider/lcov.info",
+    "pallet-market/lcov.info",
+    "mater/lcov.info"
+  ]
 }

Cargo.toml

@@ -17,6 +17,7 @@ members = [
   "pallets/proofs",
   "pallets/storage-provider",
   "primitives/proofs",
+  "primitives/shared",
   "runtime",
   "storage/mater",
   "storage/mater-cli",
@@ -126,6 +127,7 @@ pallet-storage-provider = { path = "pallets/storage-provider", default-features
 polka-storage-proofs = { path = "lib/polka-storage-proofs", default-features = false }
 polka-storage-runtime = { path = "runtime" }
 primitives-proofs = { path = "primitives/proofs", default-features = false }
+primitives-shared = { path = "primitives/shared" }
 storagext = { path = "cli/polka-storage/storagext" }
 
 # FileCoin proofs

cli/polka-storage-provider/Cargo.toml

@@ -18,10 +18,10 @@ codec = { workspace = true }
 filecoin-hashers.workspace = true
 fr32.workspace = true
 futures = { workspace = true }
-ipld-core.workspace = true
 jsonrpsee = { workspace = true, features = ["http-client", "macros", "server", "ws-client"] }
 polka-storage-proofs = { workspace = true, features = ["std", "substrate"] }
 primitives-proofs = { workspace = true, features = ["std"] }
+primitives-shared = { workspace = true }
 sc-cli = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }

cli/polka-storage-provider/src/commands/utils/commp.rs

@@ -1,36 +1,28 @@
-use std::io::{BufReader, Read};
+use std::io::Read;
 
 use filecoin_hashers::{
     sha256::{Sha256Domain, Sha256Hasher},
     Domain,
 };
-use fr32::{to_padded_bytes, Fr32Reader};
-use ipld_core::cid::multihash::Multihash;
-use mater::Cid;
-use storage_proofs_core::{merkle::BinaryMerkleTree, util::NODE_SIZE};
+use fr32::Fr32Reader;
+use primitives_shared::{
+    commitment::{Commitment, CommitmentKind},
+    piece::PaddedPieceSize,
+    NODE_SIZE,
+};
+use storage_proofs_core::merkle::BinaryMerkleTree;
 use thiserror::Error;
 
-/// SHA2-256 with the two most significant bits from the last byte zeroed (as
-/// via a mask with 0b00111111) - used for proving trees as in Filecoin.
-///
-/// https://github.com/multiformats/multicodec/blob/badcfe56bb7e0bbb06b60d57565186cd6be1f932/table.csv#L153
-pub const SHA2_256_TRUNC254_PADDED: u64 = 0x1012;
-
-/// Filecoin piece or sector data commitment merkle node/root (CommP & CommD)
-///
-/// https://github.com/multiformats/multicodec/blob/badcfe56bb7e0bbb06b60d57565186cd6be1f932/table.csv#L554
-pub const FIL_COMMITMENT_UNSEALED: u64 = 0xf101;
-
 /// Reader that returns zeros if the inner reader is empty.
-struct ZeroPaddingReader<R: Read> {
+pub struct ZeroPaddingReader<R: Read> {
     /// The inner reader to read from.
     inner: R,
     /// The number of bytes this 0-padding reader has left to produce.
-    remaining: usize,
+    remaining: u64,
 }
 
 impl<R: Read> ZeroPaddingReader<R> {
-    fn new(inner: R, total_size: usize) -> Self {
+    pub fn new(inner: R, total_size: u64) -> Self {
         Self {
             inner,
             remaining: total_size,
@@ -45,7 +37,7 @@ impl<R: Read> Read for ZeroPaddingReader<R> {
         }
 
         // Number of bytes that the reader will produce in this execution
-        let to_read = buf.len().min(self.remaining);
+        let to_read = buf.len().min(self.remaining as usize);
         // Number of bytes that we read from the inner reader
         let read = self.inner.read(&mut buf[..to_read])?;
 
@@ -56,60 +48,32 @@ impl<R: Read> Read for ZeroPaddingReader<R> {
         }
 
         // Decrease the number of bytes this 0-padding reader has left to produce.
-        self.remaining -= to_read;
+        self.remaining -= to_read as u64;
 
         // Return the number of bytes that we wrote to the buffer.
         Ok(to_read)
     }
 }
 
-// Ensure that the padded piece size is valid.
-fn ensure_piece_size(padded_piece_size: usize) -> Result<(), CommPError> {
-    if padded_piece_size < NODE_SIZE {
-        return Err(CommPError::PieceTooSmall);
-    }
-
-    if padded_piece_size % NODE_SIZE != 0 {
-        return Err(CommPError::InvalidPieceSize(format!(
-            "padded_piece_size is not multiple of {NODE_SIZE}"
-        )));
-    }
-
-    Ok(())
-}
-
 /// Calculate the piece commitment for a given data source.
 ///
 ///  Reference — <https://spec.filecoin.io/systems/filecoin_files/piece/#section-systems.filecoin_files.piece.data-representation>
 pub fn calculate_piece_commitment<R: Read>(
     source: R,
-    unpadded_piece_size: u64,
-) -> Result<[u8; 32], CommPError> {
-    // Wrap the source in a BufReader for efficient reading.
-    let source = BufReader::new(source);
+    piece_size: PaddedPieceSize,
+) -> Result<Commitment, CommPError> {
     // This reader adds two zero bits to each 254 bits of data read from the source.
-    let fr32_reader = Fr32Reader::new(source);
-    // This is the padded piece size after we add 2 zero bits to each 254 bits of data.
-    let padded_piece_size = to_padded_bytes(unpadded_piece_size as usize);
-    // Final padded piece size should be 2^n where n is a positive integer. That
-    // is because we are using MerkleTree to calculate the piece commitment.
-    let padded_piece_size = padded_piece_size.next_power_of_two();
-
-    // Ensure that the piece size is valid, before generating a MerkeTree.
-    ensure_piece_size(padded_piece_size)?;
-
-    // The reader that pads the source with zeros
-    let mut zero_padding_reader = ZeroPaddingReader::new(fr32_reader, padded_piece_size);
+    let mut fr32_reader = Fr32Reader::new(source);
 
     // Buffer used for reading data used for leafs.
     let mut buffer = [0; NODE_SIZE];
     // Number of leafs
-    let num_leafs = padded_piece_size.div_ceil(NODE_SIZE) as usize;
+    let num_leafs = piece_size.div_ceil(NODE_SIZE as u64) as usize;
 
     // Elements iterator used by the MerkleTree. The elements returned by the
     // iterator represent leafs of the tree
     let elements_iterator = (0..num_leafs).map(|_| {
-        zero_padding_reader.read_exact(&mut buffer)?;
+        fr32_reader.read_exact(&mut buffer)?;
         let hash = Sha256Domain::try_from_bytes(&buffer)?;
         Ok(hash)
     });
@@ -122,14 +86,9 @@ pub fn calculate_piece_commitment<R: Read>(
         .write_bytes(&mut commitment)
         .expect("destination buffer large enough");
 
-    Ok(commitment)
-}
+    let commitment = Commitment::new(commitment, CommitmentKind::Piece);
 
-/// Generate Cid from the piece commitment
-pub fn piece_commitment_cid(piece_commitment: [u8; 32]) -> Cid {
-    let hash = Multihash::wrap(SHA2_256_TRUNC254_PADDED, &piece_commitment)
-        .expect("piece commitment not more than 64 bytes");
-    Cid::new_v1(FIL_COMMITMENT_UNSEALED, hash)
+    Ok(commitment)
 }
 
 #[derive(Debug, Error)]
@@ -146,11 +105,12 @@ pub enum CommPError {
 
 #[cfg(test)]
 mod tests {
-    use std::io::Read;
+    use std::io::{Cursor, Read};
 
-    use crate::commands::utils::commp::{
-        calculate_piece_commitment, CommPError, ZeroPaddingReader,
-    };
+    use primitives_proofs::SectorSize;
+    use primitives_shared::piece::PaddedPieceSize;
+
+    use crate::commands::utils::commp::{calculate_piece_commitment, ZeroPaddingReader};
 
     #[test]
     fn test_zero_padding_reader() {
@@ -179,25 +139,40 @@ mod tests {
 
     #[test]
     fn test_calculate_piece_commitment() {
-        use std::io::Cursor;
-
-        let data = vec![2u8; 200];
+        let data_size: u64 = 200;
+        let data = vec![2u8; data_size as usize];
         let cursor = Cursor::new(data.clone());
+        let padded_piece_size = PaddedPieceSize::from_arbitrary_size(data_size);
+        let zero_padding_reader = ZeroPaddingReader::new(cursor, *padded_piece_size);
 
-        let commitment = calculate_piece_commitment(cursor, data.len() as u64).unwrap();
+        let commitment =
+            calculate_piece_commitment(zero_padding_reader, padded_piece_size).unwrap();
         assert_eq!(
-            commitment,
+            commitment.raw(),
             [
                 152, 58, 157, 235, 187, 58, 81, 61, 113, 252, 178, 149, 158, 13, 242, 24, 54, 98,
                 148, 15, 250, 217, 3, 24, 152, 110, 93, 173, 117, 209, 251, 37,
             ]
         );
+    }
 
-        // Test with zero-length data
-        let empty_data = Vec::new();
-        let empty_cursor = Cursor::new(empty_data);
+    #[test]
+    fn test_zero_piece_commitment() {
+        let size = SectorSize::_2KiB;
+        let padded_piece_size = PaddedPieceSize::new(size.bytes()).unwrap();
+        let cursor = Cursor::new(vec![]);
+        let zero_padding_reader = ZeroPaddingReader::new(cursor, *padded_piece_size);
+
+        let commitment =
+            calculate_piece_commitment(zero_padding_reader, padded_piece_size).unwrap();
+        dbg!(commitment.raw());
 
-        let empty_commitment = calculate_piece_commitment(empty_cursor, 0);
-        assert!(matches!(empty_commitment, Err(CommPError::PieceTooSmall)));
+        assert_eq!(
+            commitment.raw(),
+            [
+                252, 126, 146, 130, 150, 229, 22, 250, 173, 233, 134, 178, 143, 146, 212, 74, 79,
+                36, 185, 53, 72, 82, 35, 55, 106, 121, 144, 39, 188, 24, 248, 51
+            ]
+        );
     }
 }

cli/polka-storage-provider/src/commands/utils/mod.rs

@@ -1,13 +1,18 @@
 mod commp;
 
-use std::{fs::File, io::Write, path::PathBuf};
+use std::{
+    fs::File,
+    io::{BufReader, Write},
+    path::PathBuf,
+};
 
 use mater::CarV2Reader;
 use polka_storage_proofs::porep;
 use primitives_proofs::RegisteredSealProof;
+use primitives_shared::piece::PaddedPieceSize;
 
 use crate::{
-    commands::utils::commp::{calculate_piece_commitment, piece_commitment_cid, CommPError},
+    commands::utils::commp::{calculate_piece_commitment, CommPError, ZeroPaddingReader},
     CliError,
 };
 
@@ -44,18 +49,24 @@ impl UtilsCommand {
                 car_v2_reader.is_car_file().await?;
 
                 // Calculate the piece commitment.
-                let mut source_file = File::open(&input_path)?;
+                let source_file = File::open(&input_path)?;
                 let file_size = source_file.metadata()?.len();
 
+                let buffered = BufReader::new(source_file);
+                let padded_piece_size = PaddedPieceSize::from_arbitrary_size(file_size as u64);
+                let mut zero_padding_reader = ZeroPaddingReader::new(buffered, *padded_piece_size);
+
                 // The calculate_piece_commitment blocks the thread. We could
                 // use tokio::task::spawn_blocking to avoid this, but in this
                 // case it doesn't matter because this is the only thing we are
                 // working on.
-                let commitment = calculate_piece_commitment(&mut source_file, file_size)
-                    .map_err(|err| UtilsCommandError::CommPError(err))?;
-                let cid = piece_commitment_cid(commitment);
+                let commitment =
+                    calculate_piece_commitment(&mut zero_padding_reader, padded_piece_size)
+                        .map_err(|err| UtilsCommandError::CommPError(err))?;
+                let cid = commitment.cid();
 
                 println!("Piece commitment CID: {cid}");
+                println!("Padded size: {padded_piece_size}");
             }
             UtilsCommand::GeneratePoRepParams {
                 seal_proof,
@@ -106,6 +117,8 @@ impl UtilsCommand {
 pub enum UtilsCommandError {
     #[error("the commp command failed because: {0}")]
     CommPError(#[from] CommPError),
+    #[error("CidError: {0}")]
+    CidError(String),
     #[error("failed to create a file '{0}' because: {1}")]
     FileCreateError(PathBuf, std::io::Error),
     #[error("failed to convert from rust-fil-proofs to polka-storage-proofs: {0}")]

pallets/market/Cargo.toml

@@ -22,7 +22,9 @@ hex = { workspace = true, default-features = false, features = ["alloc"] }
 log = { workspace = true }
 multihash-codetable = { workspace = true, features = ["blake2b"] }
 primitives-proofs = { workspace = true, default-features = false }
+primitives-shared = { workspace = true }
 scale-info = { workspace = true, default-features = false, features = ["derive"] }
+sha2 = { workspace = true }
 
 # frame deps
 frame-benchmarking = { workspace = true, default-features = false, optional = true }