feat(commd): calculate commd (#423)

eigerco · Oct 9, 2024 · 7541a46 · 7541a46
1 parent 9433eb8
commit 7541a46
Show file tree

Hide file tree

Showing 24 changed files with 1,262 additions and 178 deletions.
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -1,11 +1,11 @@
 {
-    "files.trimTrailingWhitespace": true,
-    "rust-analyzer.rustfmt.extraArgs": ["+nightly"],
-    "rust-analyzer.checkOnSave": false,
-    "coverage-gutters.coverageBaseDir": "coverage",
-    "coverage-gutters.coverageFileNames": [
-        "pallet-storage-provider/lcov.info",
-        "pallet-market/lcov.info",
-        "mater/lcov.info"
-    ]
+  "files.trimTrailingWhitespace": true,
+  "rust-analyzer.rustfmt.extraArgs": ["+nightly"],
+  "rust-analyzer.checkOnSave": false,
+  "coverage-gutters.coverageBaseDir": "coverage",
+  "coverage-gutters.coverageFileNames": [
+    "pallet-storage-provider/lcov.info",
+    "pallet-market/lcov.info",
+    "mater/lcov.info"
+  ]
 }
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -16,6 +16,7 @@ members = [
   "pallets/market",
   "pallets/proofs",
   "pallets/storage-provider",
+  "primitives/commitment",
   "primitives/proofs",
   "runtime",
   "storage/mater",
@@ -126,6 +127,7 @@ pallet-proofs = { path = "pallets/proofs", default-features = false }
 pallet-storage-provider = { path = "pallets/storage-provider", default-features = false }
 polka-storage-proofs = { path = "lib/polka-storage-proofs", default-features = false }
 polka-storage-runtime = { path = "runtime" }
+primitives-commitment = { path = "primitives/commitment" }
 primitives-proofs = { path = "primitives/proofs", default-features = false }
 storagext = { path = "cli/polka-storage/storagext" }
 

diff --git a/cli/artifacts/metadata.scale b/cli/artifacts/metadata.scale
diff --git a/cli/polka-storage-provider/Cargo.toml b/cli/polka-storage-provider/Cargo.toml
@@ -23,9 +23,9 @@ filecoin-hashers.workspace = true
 fr32.workspace = true
 futures = { workspace = true }
 hex = { workspace = true }
-ipld-core.workspace = true
 jsonrpsee = { workspace = true, features = ["http-client", "macros", "server", "ws-client"] }
 polka-storage-proofs = { workspace = true, features = ["std", "substrate"] }
+primitives-commitment = { workspace = true }
 primitives-proofs = { workspace = true, features = ["std"] }
 rand = { workspace = true }
 rocksdb = { workspace = true }

diff --git a/cli/polka-storage-provider/src/commands/utils.rs b/cli/polka-storage-provider/src/commands/utils.rs
@@ -1,14 +1,20 @@
-use std::{fs::File, io::Write, path::PathBuf, str::FromStr};
+use std::{
+    fs::File,
+    io::{BufReader, Write},
+    path::PathBuf,
+    str::FromStr,
+};
 
 use mater::CarV2Reader;
 use polka_storage_proofs::{
     porep::{self, sealer::Sealer},
     types::PieceInfo,
 };
+use primitives_commitment::piece::PaddedPieceSize;
 use primitives_proofs::RegisteredSealProof;
 
 use crate::{
-    commp::{calculate_piece_commitment, piece_commitment_cid, CommPError},
+    commp::{calculate_piece_commitment, CommPError, ZeroPaddingReader},
     CliError,
 };
 
@@ -67,18 +73,24 @@ impl UtilsCommand {
                     .map_err(|e| UtilsCommandError::InvalidCARv2(input_path.clone(), e))?;
 
                 // Calculate the piece commitment.
-                let mut source_file = File::open(&input_path)?;
+                let source_file = File::open(&input_path)?;
                 let file_size = source_file.metadata()?.len();
 
+                let buffered = BufReader::new(source_file);
+                let padded_piece_size = PaddedPieceSize::from_arbitrary_size(file_size as u64);
+                let mut zero_padding_reader = ZeroPaddingReader::new(buffered, *padded_piece_size);
+
                 // The calculate_piece_commitment blocks the thread. We could
                 // use tokio::task::spawn_blocking to avoid this, but in this
                 // case it doesn't matter because this is the only thing we are
                 // working on.
-                let commitment = calculate_piece_commitment(&mut source_file, file_size)
-                    .map_err(|err| UtilsCommandError::CommPError(err))?;
-                let cid = piece_commitment_cid(commitment);
+                let commitment =
+                    calculate_piece_commitment(&mut zero_padding_reader, padded_piece_size)
+                        .map_err(|err| UtilsCommandError::CommPError(err))?;
+                let cid = commitment.cid();
 
-                println!("{cid}");
+                println!("Piece commitment CID: {cid}");
+                println!("Padded size: {padded_piece_size}");
             }
             UtilsCommand::GeneratePoRepParams {
                 seal_proof,

diff --git a/cli/polka-storage-provider/src/commp.rs b/cli/polka-storage-provider/src/commp.rs
@@ -1,36 +1,24 @@
-use std::io::{BufReader, Read};
+use std::io::Read;
 
 use filecoin_hashers::{
     sha256::{Sha256Domain, Sha256Hasher},
     Domain,
 };
-use fr32::{to_padded_bytes, Fr32Reader};
-use ipld_core::cid::multihash::Multihash;
-use mater::Cid;
-use storage_proofs_core::{merkle::BinaryMerkleTree, util::NODE_SIZE};
+use fr32::Fr32Reader;
+use primitives_commitment::{piece::PaddedPieceSize, Commitment, CommitmentKind, NODE_SIZE};
+use storage_proofs_core::merkle::BinaryMerkleTree;
 use thiserror::Error;
 
-/// SHA2-256 with the two most significant bits from the last byte zeroed (as
-/// via a mask with 0b00111111) - used for proving trees as in Filecoin.
-///
-/// https://github.com/multiformats/multicodec/blob/badcfe56bb7e0bbb06b60d57565186cd6be1f932/table.csv#L153
-pub const SHA2_256_TRUNC254_PADDED: u64 = 0x1012;
-
-/// Filecoin piece or sector data commitment merkle node/root (CommP & CommD)
-///
-/// https://github.com/multiformats/multicodec/blob/badcfe56bb7e0bbb06b60d57565186cd6be1f932/table.csv#L554
-pub const FIL_COMMITMENT_UNSEALED: u64 = 0xf101;
-
 /// Reader that returns zeros if the inner reader is empty.
 pub struct ZeroPaddingReader<R: Read> {
     /// The inner reader to read from.
     inner: R,
     /// The number of bytes this 0-padding reader has left to produce.
-    remaining: usize,
+    remaining: u64,
 }
 
 impl<R: Read> ZeroPaddingReader<R> {
-    pub fn new(inner: R, total_size: usize) -> Self {
+    pub fn new(inner: R, total_size: u64) -> Self {
         Self {
             inner,
             remaining: total_size,
@@ -45,7 +33,7 @@ impl<R: Read> Read for ZeroPaddingReader<R> {
         }
 
         // Number of bytes that the reader will produce in this execution
-        let to_read = buf.len().min(self.remaining);
+        let to_read = buf.len().min(self.remaining as usize);
         // Number of bytes that we read from the inner reader
         let read = self.inner.read(&mut buf[..to_read])?;
 
@@ -56,60 +44,32 @@ impl<R: Read> Read for ZeroPaddingReader<R> {
         }
 
         // Decrease the number of bytes this 0-padding reader has left to produce.
-        self.remaining -= to_read;
+        self.remaining -= to_read as u64;
 
         // Return the number of bytes that we wrote to the buffer.
         Ok(to_read)
     }
 }
 
-// Ensure that the padded piece size is valid.
-pub fn ensure_piece_size(padded_piece_size: usize) -> Result<(), CommPError> {
-    if padded_piece_size < NODE_SIZE {
-        return Err(CommPError::PieceTooSmall);
-    }
-
-    if padded_piece_size % NODE_SIZE != 0 {
-        return Err(CommPError::InvalidPieceSize(format!(
-            "padded_piece_size is not multiple of {NODE_SIZE}"
-        )));
-    }
-
-    Ok(())
-}
-
 /// Calculate the piece commitment for a given data source.
 ///
 ///  Reference — <https://spec.filecoin.io/systems/filecoin_files/piece/#section-systems.filecoin_files.piece.data-representation>
 pub fn calculate_piece_commitment<R: Read>(
     source: R,
-    unpadded_piece_size: u64,
-) -> Result<[u8; 32], CommPError> {
-    // Wrap the source in a BufReader for efficient reading.
-    let source = BufReader::new(source);
+    piece_size: PaddedPieceSize,
+) -> Result<Commitment, CommPError> {
     // This reader adds two zero bits to each 254 bits of data read from the source.
-    let fr32_reader = Fr32Reader::new(source);
-    // This is the padded piece size after we add 2 zero bits to each 254 bits of data.
-    let padded_piece_size = to_padded_bytes(unpadded_piece_size as usize);
-    // Final padded piece size should be 2^n where n is a positive integer. That
-    // is because we are using MerkleTree to calculate the piece commitment.
-    let padded_piece_size = padded_piece_size.next_power_of_two();
-
-    // Ensure that the piece size is valid, before generating a MerkeTree.
-    ensure_piece_size(padded_piece_size)?;
-
-    // The reader that pads the source with zeros
-    let mut zero_padding_reader = ZeroPaddingReader::new(fr32_reader, padded_piece_size);
+    let mut fr32_reader = Fr32Reader::new(source);
 
     // Buffer used for reading data used for leafs.
     let mut buffer = [0; NODE_SIZE];
     // Number of leafs
-    let num_leafs = padded_piece_size.div_ceil(NODE_SIZE) as usize;
+    let num_leafs = piece_size.div_ceil(NODE_SIZE as u64) as usize;
 
     // Elements iterator used by the MerkleTree. The elements returned by the
     // iterator represent leafs of the tree
     let elements_iterator = (0..num_leafs).map(|_| {
-        zero_padding_reader.read_exact(&mut buffer)?;
+        fr32_reader.read_exact(&mut buffer)?;
         let hash = Sha256Domain::try_from_bytes(&buffer)?;
         Ok(hash)
     });
@@ -122,20 +82,13 @@ pub fn calculate_piece_commitment<R: Read>(
         .write_bytes(&mut commitment)
         .expect("destination buffer large enough");
 
-    Ok(commitment)
-}
+    let commitment = Commitment::new(commitment, CommitmentKind::Piece);
 
-/// Generate Cid from the piece commitment
-pub fn piece_commitment_cid(piece_commitment: [u8; 32]) -> Cid {
-    let hash = Multihash::wrap(SHA2_256_TRUNC254_PADDED, &piece_commitment)
-        .expect("piece commitment not more than 64 bytes");
-    Cid::new_v1(FIL_COMMITMENT_UNSEALED, hash)
+    Ok(commitment)
 }
 
 #[derive(Debug, Error)]
 pub enum CommPError {
-    #[error("Piece is too small")]
-    PieceTooSmall,
     #[error("Piece is not valid size: {0}")]
     InvalidPieceSize(String),
     #[error("Tree build error: {0}")]
@@ -146,9 +99,12 @@ pub enum CommPError {
 
 #[cfg(test)]
 mod tests {
-    use std::io::Read;
+    use std::io::{Cursor, Read};
 
-    use super::{calculate_piece_commitment, CommPError, ZeroPaddingReader};
+    use primitives_commitment::piece::PaddedPieceSize;
+    use primitives_proofs::SectorSize;
+
+    use super::{calculate_piece_commitment, ZeroPaddingReader};
 
     #[test]
     fn test_zero_padding_reader() {
@@ -177,25 +133,40 @@ mod tests {
 
     #[test]
     fn test_calculate_piece_commitment() {
-        use std::io::Cursor;
-
-        let data = vec![2u8; 200];
+        let data_size: u64 = 200;
+        let data = vec![2u8; data_size as usize];
         let cursor = Cursor::new(data.clone());
+        let padded_piece_size = PaddedPieceSize::from_arbitrary_size(data_size);
+        let zero_padding_reader = ZeroPaddingReader::new(cursor, *padded_piece_size);
 
-        let commitment = calculate_piece_commitment(cursor, data.len() as u64).unwrap();
+        let commitment =
+            calculate_piece_commitment(zero_padding_reader, padded_piece_size).unwrap();
         assert_eq!(
-            commitment,
+            commitment.raw(),
             [
                 152, 58, 157, 235, 187, 58, 81, 61, 113, 252, 178, 149, 158, 13, 242, 24, 54, 98,
                 148, 15, 250, 217, 3, 24, 152, 110, 93, 173, 117, 209, 251, 37,
             ]
         );
+    }
 
-        // Test with zero-length data
-        let empty_data = Vec::new();
-        let empty_cursor = Cursor::new(empty_data);
+    #[test]
+    fn test_zero_piece_commitment() {
+        let size = SectorSize::_2KiB;
+        let padded_piece_size = PaddedPieceSize::new(size.bytes()).unwrap();
+        let cursor = Cursor::new(vec![]);
+        let zero_padding_reader = ZeroPaddingReader::new(cursor, *padded_piece_size);
+
+        let commitment =
+            calculate_piece_commitment(zero_padding_reader, padded_piece_size).unwrap();
+        dbg!(commitment.raw());
 
-        let empty_commitment = calculate_piece_commitment(empty_cursor, 0);
-        assert!(matches!(empty_commitment, Err(CommPError::PieceTooSmall)));
+        assert_eq!(
+            commitment.raw(),
+            [
+                252, 126, 146, 130, 150, 229, 22, 250, 173, 233, 134, 178, 143, 146, 212, 74, 79,
+                36, 185, 53, 72, 82, 35, 55, 106, 121, 144, 39, 188, 24, 248, 51
+            ]
+        );
     }
 }
diff --git a/cli/polka-storage-provider/src/storage.rs b/cli/polka-storage-provider/src/storage.rs
@@ -10,6 +10,7 @@ use axum::{
 };
 use futures::{TryFutureExt, TryStreamExt};
 use mater::Cid;
+use primitives_commitment::piece::PaddedPieceSize;
 use tokio::{
     fs::{self, File},
     io::{AsyncRead, BufWriter},
@@ -22,7 +23,7 @@ use tower_http::trace::TraceLayer;
 use uuid::Uuid;
 
 use crate::{
-    commp::{calculate_piece_commitment, piece_commitment_cid, CommPError},
+    commp::{calculate_piece_commitment, CommPError},
     db::DealDB,
 };
 
@@ -206,8 +207,9 @@ async fn upload(
     let piece_commitment_cid = tokio::task::spawn_blocking(move || -> Result<_, CommPError> {
         let file = std::fs::File::open(&piece_path).map_err(CommPError::Io)?;
         let file_size = file.metadata().map_err(CommPError::Io)?.len();
+        let file_size = PaddedPieceSize::new(file_size).map_err(|err| CommPError::InvalidPieceSize(err.to_string()))?;
         let piece_commitment = calculate_piece_commitment(file, file_size)?;
-        let piece_commitment_cid = piece_commitment_cid(piece_commitment);
+        let piece_commitment_cid = piece_commitment.cid();
         tracing::debug!(path = %piece_path.display(), commp = %piece_commitment_cid, "calculated piece commitment");
         Ok(piece_commitment_cid)
     })