Merge branch 'develop' into chore/578/general-review-project-structure

Cargo.toml

@@ -106,7 +106,7 @@ subxt = { version = "0.38.0" }
 subxt-signer = "0.38.0"
 syn = { version = "2.0.53" }
 tempfile = "3.10.1"
-thiserror = { version = "2.0.3" }
+thiserror = { version = "2.0.3", default-features = false }
 tokio = "1.37.0"
 tokio-stream = "0.1.15"
 tokio-util = "0.7.11"

README.md

@@ -1,5 +1,12 @@
 # Polka Storage
 
+> [!NOTE]
+> The stable branch is [`main`](https://github.com/eigerco/polka-storage/tree/main),
+> while the default GitHub branch is [`develop`](https://github.com/eigerco/polka-storage/tree/develop).
+>
+> We aim to keep Polka Storage Book synced with the `main` branch,
+> if you notice discrepancies, please [file an issue](https://github.com/eigerco/polka-storage/issues/new).
+
 Welcome to the Polka Storage project repo!
 
 This project aims to build a native storage network for Polkadot.

cli/polka-storage-provider/client/Cargo.toml

@@ -21,6 +21,7 @@ bls12_381 = { workspace = true }
 cid = { workspace = true, features = ["std"] }
 clap = { workspace = true, features = ["derive"] }
 codec = { workspace = true }
+hex = { workspace = true }
 jsonrpsee = { workspace = true, features = ["http-client", "macros", "server", "ws-client"] }
 sc-cli = { workspace = true }
 serde = { workspace = true }

cli/polka-storage-provider/client/src/commands/proofs.rs

@@ -5,6 +5,7 @@ use std::{
     str::FromStr,
 };
 
+use codec::Encode;
 use mater::CarV2Reader;
 use polka_storage_proofs::{
     porep::{self, sealer::Sealer},
@@ -14,9 +15,13 @@ use polka_storage_proofs::{
 use polka_storage_provider_common::commp::{calculate_piece_commitment, CommPError};
 use primitives_commitment::{
     piece::{PaddedPieceSize, PieceInfo},
-    Commitment,
+    Commitment, CommitmentError,
+};
+use primitives_proofs::{
+    derive_prover_id,
+    randomness::{draw_randomness, DomainSeparationTag},
+    RegisteredPoStProof, RegisteredSealProof, SectorNumber,
 };
-use primitives_proofs::{derive_prover_id, RegisteredPoStProof, RegisteredSealProof};
 use storagext::multipair::{MultiPairArgs, MultiPairSigner};
 use subxt::tx::Signer;
 
@@ -72,6 +77,15 @@ pub enum ProofsCommand {
         /// Directory where the proof files and the sector will be put. Defaults to the current directory.
         #[arg(short, long)]
         output_path: Option<PathBuf>,
+        /// Sector number
+        #[arg(long)]
+        sector_id: u32,
+        /// The height at which we draw the randomness for deriving a sealed cid.
+        #[arg(long)]
+        seal_randomness_height: u64,
+        /// Precommit block number
+        #[arg(long)]
+        pre_commit_block_number: u64,
     },
     /// Generates PoSt verifying key and proving parameters for zk-SNARK workflows (submit windowed PoSt)
     #[clap(name = "post-params")]
@@ -203,17 +217,42 @@ impl ProofsCommand {
                 commp,
                 output_path,
                 cache_directory,
+                sector_id,
+                seal_randomness_height,
+                pre_commit_block_number,
             } => {
                 let Some(signer) = Option::<MultiPairSigner>::from(signer_key) else {
                     return Err(UtilsCommandError::NoSigner)?;
                 };
-                let prover_id = derive_prover_id(signer.account_id());
 
-                // Those are hardcoded for the showcase only.
-                // They should come from Storage Provider Node, precommits and other information.
-                let sector_id = 77.into();
-                let ticket = [12u8; 32];
-                let seed = [13u8; 32];
+                let sector_number = SectorNumber::try_from(sector_id)
+                    .map_err(|_| UtilsCommandError::InvalidSectorId)?;
+
+                let entropy = signer.account_id().encode();
+                println!("Entropy: {}", hex::encode(&entropy));
+
+                let ticket = get_randomness(
+                    DomainSeparationTag::SealRandomness,
+                    seal_randomness_height,
+                    &entropy,
+                );
+                println!(
+                    "[{seal_randomness_height}] Ticket randomness: {}",
+                    hex::encode(ticket)
+                );
+
+                // The number added is configured in runtime:
+                // https://github.com/eigerco/polka-storage/blob/18207759d7c6c175916d5bed70246d94a8f028f4/runtime/src/configs/mod.rs#L360
+                let interactive_block_number = pre_commit_block_number + 10;
+                let seed = get_randomness(
+                    DomainSeparationTag::InteractiveSealChallengeSeed,
+                    interactive_block_number,
+                    &entropy,
+                );
+                println!(
+                    "[{interactive_block_number}] Seed randomness: {}",
+                    hex::encode(seed)
+                );
 
                 let output_path = if let Some(output_path) = output_path {
                     output_path
@@ -273,14 +312,17 @@ impl ProofsCommand {
                     .create_sector(vec![(piece_file, piece_info)], unsealed_sector)
                     .map_err(|e| UtilsCommandError::GeneratePoRepError(e))?;
 
+                let prover_id = derive_prover_id(signer.account_id());
+                println!("Prover ID: {}", hex::encode(prover_id));
+
                 println!("Precommitting...");
                 let precommit = sealer
                     .precommit_sector(
                         &cache_directory,
                         unsealed_sector_path,
                         &sealed_sector_path,
                         prover_id,
-                        sector_id,
+                        sector_number,
                         ticket,
                         &piece_infos,
                     )
@@ -293,7 +335,7 @@ impl ProofsCommand {
                         &cache_directory,
                         &sealed_sector_path,
                         prover_id,
-                        sector_id,
+                        sector_number,
                         ticket,
                         Some(seed),
                         precommit,
@@ -310,8 +352,10 @@ impl ProofsCommand {
                     .clone()
                     .try_into()
                     .expect("converstion between rust-fil-proofs and polka-storage-proofs to work");
-                proof_scale_file.write_all(&codec::Encode::encode(&proof_scale))?;
+                let scale_encoded_proof = codec::Encode::encode(&proof_scale);
+                proof_scale_file.write_all(&scale_encoded_proof)?;
 
+                println!("Proof as HEX: {}", hex::encode(scale_encoded_proof));
                 println!("Wrote proof to {}", proof_scale_filename.display());
             }
             ProofsCommand::GeneratePoStParams {
@@ -444,8 +488,10 @@ pub enum UtilsCommandError {
     InvalidPieceFile(PathBuf, std::io::Error),
     #[error("provided invalid CommP {0}, error: {1}")]
     InvalidPieceCommP(String, cid::Error),
-    #[error("invalid piece type")]
-    InvalidPieceType(String, &'static str),
+    #[error("invalid piece type, error: {1}")]
+    InvalidPieceType(String, CommitmentError),
+    #[error("invalid sector id")]
+    InvalidSectorId,
     #[error("file {0} is invalid CARv2 file {1}")]
     InvalidCARv2(PathBuf, mater::Error),
     #[error("no signer key was provider")]
@@ -465,3 +511,13 @@ fn file_with_extension(
         .map_err(|e| UtilsCommandError::FileCreateError(new_path.clone(), e))?;
     Ok((new_path, file))
 }
+
+fn get_randomness(
+    personalization: DomainSeparationTag,
+    block_number: u64,
+    entropy: &[u8],
+) -> [u8; 32] {
+    // This randomness digest is hardcoded because it's always same on testnet.
+    let digest = [0u8; 32];
+    draw_randomness(&digest, personalization, block_number, &entropy)
+}

cli/polka-storage-provider/server/src/rpc.rs

@@ -50,6 +50,14 @@ impl StorageProviderRpcServer for RpcServerState {
             ));
         }
 
+        // TODO(@jmg-duarte,25/11/2024): Add sanity validations
+        // end_block > start_block
+        // available balance (sp & client)
+        // the provider matches us
+        // storage price per block > 0
+        // piece size <= 2048 && power of two
+        // check the piece_cid code
+
         let cid = self
             .deal_db
             .add_accepted_proposed_deal(&deal)
@@ -80,7 +88,7 @@ impl StorageProviderRpcServer for RpcServerState {
             .is_none()
         {
             return Err(RpcError::internal_error(
-                "proposal has not been accepted",
+                "proposal has not been found — have you proposed the deal first?",
                 None,
             ));
         }
@@ -95,6 +103,8 @@ impl StorageProviderRpcServer for RpcServerState {
             ));
         }
 
+        // TODO(@jmg-duarte,25/11/2024): don't batch the deals for better errors
+
         let deal_proposal = deal.deal_proposal.clone();
         // TODO(@jmg-duarte,#428,04/10/2024):
         // There's a small bug here, currently, xt_client waits for a "full extrisic submission"

docs/src/architecture/pallets/market.md

@@ -211,7 +211,6 @@ The Market Pallet actions can fail with following errors:
 - `InsufficientFreeFunds` - Market participants do not have enough free funds.
 - `NoProposalsToBePublished` - `publish_storage_deals` was called with an empty list of `deals`.
 - `ProposalsPublishedByIncorrectStorageProvider` - Is returned when calling `publish_storage_deals` and the deals in a list are not published by the same storage provider.
-- `AllProposalsInvalid` - `publish_storage_deals` call was supplied with a list of `deals` which are all invalid.
 - `DuplicateDeal` - There is more than one deal with this ID in the Sector.
 - `DealNotFound` - Tried to activate a deal that is not in the system.
 - `DealActivationError` - Tried to activate a deal, but data was malformed.
@@ -222,6 +221,23 @@ The Market Pallet actions can fail with following errors:
   - Deal is not pending.
 - `DealsTooLargeToFitIntoSector` - Sum of all deals piece sizes for a sector exceeds sector size. The sector size is based on the registered proof type. We currently only support registered `StackedDRG2KiBV1P1` proofs, which have 2KiB sector sizes.
 - `TooManyDealsPerBlock` - Tried to activate too many deals at a given `start_block`.
+- `StorageProviderNotRegistered` - An account tries to call `publish_storage_deals` but is not registered as a storage provider.
+- `CommD` - An error occurred when trying to calculate CommD.
+- `TooManyPendingDeals` - A storage provider tried to propose a deal, but there are too many pending deals. The pending deals should be activated or wait for expiry.
+- `InvalidProvider` - A deal was tried to be activated by a provider which does not own it.
+- `StartBlockElapsed` - A storage provider tries to activate a deal after the start block.
+- `SectorExpiresBeforeDeal` - Sector containing the deal will expire before the deal is supposed to end.
+- `InvalidDealState` - A deal was attempted to be activated twice.
+- `DealNotPending` - A storage provider tried to activate a deal which is not in the pending proposals.
+- `WrongClientSignatureOnProposal` - The client signature did not match the client's public key and data.
+- `DealEndBeforeStart` - A deal was attempted to be published but the end block is before the start block and the deal is rejected.
+- `DealStartExpired` - A deal was attempted to be published but the start block is in the past and the deal is rejected.
+- `DealNotPublished` - A deal was attempted to be published but is not in the correct state.
+- `DealDurationOutOfBounds` - A deal was attempted to be published but the duration is not between [MinDealDuration](#constants) and [MaxDealDuration](#constants).
+- `InvalidPieceCid` - The deal trying to be published has an invalid piece Cid.
+- `DealIsNotActive` - When a sector is being terminated but the deal state is not active. This is the result of a programmer bug. Please [report an issue](https://github.com/eigerco/polka-storage-book/issues/new) to the developers.
+- `InvalidCaller` - A deal was found that does not belong to the storage provider. This is the result of a programmer bug. Please [report an issue](https://github.com/eigerco/polka-storage-book/issues/new) to the developers.
+- `DealNotFound` - A deal was attempted to be fetched but could not be found. This is the result of a programmer bug. Please [report an issue](https://github.com/eigerco/polka-storage-book/issues/new) to the developers.
 - `UnexpectedValidationError` - `publish_storage_deals`'s core logic was invoked with a broken invariant that should be called by `validate_deals`. Please [report an issue](https://github.com/eigerco/polka-storage-book/issues/new) to the developers.
 - `DealPreconditionFailed` - Due to a programmer bug. Please [report an issue](https://github.com/eigerco/polka-storage-book/issues/new) to the developers.
 

docs/src/getting-started/demo-file-store.md

@@ -14,6 +14,12 @@ Alice is a [Storage User](../glossary.md#storage-user) and wants to store an ima
 Alice knows that she needs to prepare an image for storage and get its [CID](https://github.com/multiformats/cid).
 To do so, she first converts it into a [CARv2 archive](https://ipld.io/specs/transport/car/carv2/) and gets the piece cid.
 
+<div class="warning">
+
+**🚧 Currently, the maximum supported piece size is 2048 bytes!**
+
+</div>
+
 ```bash
 $ mater-cli convert -q --overwrite polkadot.svg polkadot.car
 bafkreihoxd7eg2domoh2fxqae35t7ihbonyzcdzh5baevxzrzkaakevuvy
@@ -112,9 +118,6 @@ To sign her deal proposal she runs:
 
 ```bash
 $ polka-storage-provider-client sign-deal --sr25519-key "//Alice" @polka-logo-deal.json
-```
-
-```json
 {
   "deal_proposal": {
     "piece_cid": "baga6ea4seaqabpfwrqjcwrb4pxmo2d3dyrgj24kt4vqqqcbjoph4flpj2e5lyoq",
@@ -147,3 +150,17 @@ Successfully published deal of id: 0
 ```
 
 On Alice's side, that's it!
+
+> Note that you **must** sign and submit the deals before the `start_block`!
+>
+> Currently, the storage provider **does not reject deal proposals** on the basis of how early or late they are,
+> however, this may change! ⚠️ — for example, if a storage provider knows that they won't have enough time to process
+> your file for proof submission, they may refuse your proposal on the basis that if they accept it, they may fail the deadline.
+
+## Additional Notes
+
+* As other parts of this project, file retrieval is actively being worked on! 🚧
+* Files stored in storage providers are public, as such, we suggest you encrypt your files.
+  While file encryption is a broad enough topic, if you not sure about which tools to use,
+  we suggest you keep it simple by compressing your file in a format such as [7zip](https://www.7-zip.org/) and using its encryption features.
+  If 7zip doesn't cut it, you may want to take a look into [age](https://github.com/FiloSottile/age) or [VeraCrypt](https://www.veracrypt.fr/code/VeraCrypt/).

lib/polka-storage-proofs/src/porep/sealer.rs

@@ -217,7 +217,7 @@ impl Sealer {
                 unsealed_sector,
                 sealed_sector,
                 prover_id,
-                sector_id.into(),
+                storage_proofs_core::sector::SectorId::from(u64::from(sector_id)),
                 ticket,
                 &piece_infos,
             )?;
@@ -278,7 +278,7 @@ impl Sealer {
                 cache_path,
                 replica_path,
                 prover_id,
-                sector_id.into(),
+                storage_proofs_core::sector::SectorId::from(u64::from(sector_id)),
                 ticket,
                 seed,
                 pre_commit.into(),

lib/polka-storage-proofs/src/post/mod.rs

@@ -77,7 +77,7 @@ pub fn generate_window_post<CacheDirectory: AsRef<Path>>(
     let mut replicas = BTreeMap::new();
     for replica in partition_replicas {
         replicas.insert(
-            replica.sector_id.into(),
+            storage_proofs_core::sector::SectorId::from(u64::from(replica.sector_id)),
             PrivateReplicaInfo::<Tree>::new(
                 replica.replica_path,
                 replica.comm_r,

maat/generate_porep_proof.sh

@@ -0,0 +1,22 @@
+# Clear cache before run
+CACHE_FOLDER="/tmp/psp-cache"
+rm -r "$CACHE_FOLDER"
+mkdir "$CACHE_FOLDER"
+
+PROVIDER="//Charlie"
+CAR_FILE="../examples/test-data-big.car"
+SECTOR_CID="baga6ea4seaqbfhdvmk5qygevit25ztjwl7voyikb5k2fqcl2lsuefhaqtukuiii"
+PARAMS_PATH="../2KiB.porep.params"
+SECTOR_ID=1
+SEAL_RANDOMNESS_HEIGHT=20
+PRE_COMMIT_BLOCK_NUMBER=30
+
+polka-storage-provider-client proofs porep \
+--sr25519-key "$PROVIDER" \
+--proof-parameters-path "$PARAMS_PATH" \
+--cache-directory "$CACHE_FOLDER" \
+--sector-id "$SECTOR_ID" \
+--seal-randomness-height "$SEAL_RANDOMNESS_HEIGHT" \
+--pre-commit-block-number "$PRE_COMMIT_BLOCK_NUMBER" \
+"$CAR_FILE" \
+"$SECTOR_CID"