feat: integrate filecoin-proofs fuzzers

.circleci/config.yml

@@ -20,6 +20,15 @@ jobs:
     steps:
       - configure_environment_variables
       - checkout
+      - run:
+          name: Update dependencies
+          command: apt-get update
+          no_output_timeout: 5m
+      - run:
+          name: Install dependencies
+          command: apt-get install -y clang-7 libfuzzer-7-dev libllvm7 llvm-7 llvm-7-dev llvm-7-runtime libc++-7-dev
+          no_output_timeout: 5m
+      - run: clang++-7 --version
       - run:
           name: Calculate dependencies
           command: cargo generate-lockfile
@@ -33,6 +42,7 @@ jobs:
       - run: rustup component add clippy
       - run: cargo update
       - run: cargo fetch
+      - run: cargo install cargo-fuzz
       - run: rustc +$(cat rust-toolchain) --version
       - persist_to_workspace:
           root: "."
@@ -140,6 +150,38 @@ jobs:
           command: cargo +$(cat rust-toolchain) build --benches --verbose --workspace
           no_output_timeout: 15m
 
+  fuzz_nightly:
+    docker:
+      - image: filecoin/rust:latest
+    working_directory: /mnt/crate
+    resource_class: 2xlarge+
+    steps:
+      - configure_environment_variables
+      - checkout
+      - attach_workspace:
+          at: "."
+      - restore_cache:
+          keys:
+            - cargo-v26d-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+      - restore_parameter_cache
+      - run:
+          name: Fuzz (nightly)
+          command: |
+              cd filecoin-proofs
+              cargo +$(cat rust-toolchain) fuzz run --release seal_pre_commit_phase1
+              cargo +$(cat rust-toolchain) fuzz run --release seal_pre_commit_phase2
+              cargo +$(cat rust-toolchain) fuzz run --release seal_commit_phase1
+              cargo +$(cat rust-toolchain) fuzz run --release seal_commit_phase2
+              cargo +$(cat rust-toolchain) fuzz run --release compute_d
+              cargo +$(cat rust-toolchain) fuzz run --release verify_seal
+              cargo +$(cat rust-toolchain) fuzz run --release verify_batch_seal
+              cargo +$(cat rust-toolchain) fuzz run --release get_unsealed_range
+              cargo +$(cat rust-toolchain) fuzz run --release validate_cache_for_precommit_phase2
+              cargo +$(cat rust-toolchain) fuzz run --release validate_cache_for_commit
+              cargo +$(cat rust-toolchain) fuzz run --release generate_window_post
+              cargo +$(cat rust-toolchain) fuzz run --release verify_window_post
+          no_output_timeout: 15m
+
   metrics_capture:
     docker:
       - image: filecoin/rust:latest

filecoin-proofs/Cargo.toml

@@ -56,6 +56,7 @@ generic-array = "0.13.2"
 structopt = "0.3.12"
 humansize = "1.1.0"
 indicatif = "0.14.0"
+libfuzzer-sys = { version = "0.3", features = ["arbitrary-derive"] }
 
 [dependencies.reqwest]
 version = "0.10"

filecoin-proofs/fuzz/.gitignore

@@ -0,0 +1,4 @@
+
+target
+corpus
+artifacts

filecoin-proofs/fuzz/Cargo.toml

@@ -0,0 +1,72 @@
+[package]
+name = "filecoin-proofs-fuzz"
+version = "0.0.0"
+authors = ["Automatically generated"]
+publish = false
+edition = "2018"
+
+[package.metadata]
+cargo-fuzz = true
+
+[dependencies]
+bincode = "1.1.2"
+hex = "0.4.2"
+typenum = "1.11.2"
+libfuzzer-sys = { version = "0.3", features = ["arbitrary-derive"] }
+
+[dependencies.filecoin-proofs]
+path = ".."
+[dependencies.storage-proofs]
+path = "../../storage-proofs"
+
+# Prevent this from interfering with workspaces
+[workspace]
+members = ["."]
+
+[[bin]]
+name = "generate_window_post"
+path = "fuzz_targets/generate_window_post.rs"
+
+[[bin]]
+name = "verify_window_post"
+path = "fuzz_targets/verify_window_post.rs"
+
+[[bin]]
+name = "seal_pre_commit_phase1"
+path = "fuzz_targets/seal_pre_commit_phase1.rs"
+
+[[bin]]
+name = "seal_pre_commit_phase2"
+path = "fuzz_targets/seal_pre_commit_phase2.rs"
+
+[[bin]]
+name = "seal_commit_phase1"
+path = "fuzz_targets/seal_commit_phase1.rs"
+
+[[bin]]
+name = "seal_commit_phase2"
+path = "fuzz_targets/seal_commit_phase2.rs"
+
+[[bin]]
+name = "compute_d"
+path = "fuzz_targets/compute_d.rs"
+
+[[bin]]
+name = "verify_seal"
+path = "fuzz_targets/verify_seal.rs"
+
+[[bin]]
+name = "verify_batch_seal"
+path = "fuzz_targets/verify_batch_seal.rs"
+
+[[bin]]
+name = "get_unsealed_range"
+path = "fuzz_targets/get_unsealed_range.rs"
+
+[[bin]]
+name = "validate_cache_for_precommit_phase2"
+path = "fuzz_targets/validate_cache_for_precommit_phase2.rs"
+
+[[bin]]
+name = "validate_cache_for_commit"
+path = "fuzz_targets/validate_cache_for_commit.rs"

filecoin-proofs/fuzz/README.md

@@ -0,0 +1,24 @@
+# Instructions for running fuzzer
+
+Each fuzz target can be run with:
+
+```
+$ cargo fuzz run XX
+```
+
+Where `XX` is replaced with one fuzz target in `targets/`.
+
+For speed improvements consider adding the flags `-s leaks -O`. To set Zsaniter=leaks and release mode optimisations.
+
+## Additional instructions
+
+Some fuzz targets require generation of parameters these can be generated from
+[here](https://github.com/filecoin-project/rust-fil-proofs/blob/master/filecoin-proofs/src/bin/paramcache.rs)
+by running:
+
+```
+$ cargo run --bin paramcache --release
+```
+
+This will take a very long time, so I will also recommend commenting out some `PUBLISHED_SECTOR_SIZES`.
+Comment out all the sector sizes except `SECTOR_SIZE_2_KIB`.

filecoin-proofs/fuzz/fuzz_targets/compute_d.rs

@@ -0,0 +1,20 @@
+#![no_main]
+use filecoin_proofs::{
+    compute_comm_d, SectorSize, PieceInfo
+};
+use libfuzzer_sys::arbitrary;
+use libfuzzer_sys::fuzz_target;
+
+#[derive(arbitrary::Arbitrary, Debug)]
+pub struct Wrapper {
+    piece_infos: Vec<PieceInfo>,
+}
+
+fuzz_target!(|wrap: Wrapper| {
+    let sector_size = SectorSize(2048);
+    //println!("{:?}\n", wrap);
+    let _ = compute_comm_d(
+        sector_size,
+        &wrap.piece_infos,
+    );
+});

filecoin-proofs/fuzz/fuzz_targets/generate_window_post.rs

@@ -0,0 +1,33 @@
+#![no_main]
+use filecoin_proofs::{
+    generate_window_post, Commitment, ChallengeSeed, PoStConfig, PrivateReplicaInfo, ProverId,
+};
+use filecoin_proofs::constants::*;
+use storage_proofs::sector::SectorId;
+use libfuzzer_sys::arbitrary;
+use libfuzzer_sys::fuzz_target;
+use std::collections::BTreeMap;
+use std::path::PathBuf;
+
+#[derive(arbitrary::Arbitrary, Debug)]
+pub struct Wrapper {
+    replica: PathBuf,
+    comm_r: Commitment,
+    cache_dir: PathBuf,
+    post_config: PoStConfig,
+    randomness: ChallengeSeed,
+    prover_id: ProverId,
+    // FIXME: requires Domain to be arbitrary
+    //replicas: BTreeMap<SectorId, PrivateReplicaInfo<SectorShape2KiB>>,
+}
+
+fuzz_target!(|wrap: Wrapper| {
+    /*
+    let _ = generate_window_post::<SectorShape2KiB>(
+        &wrap.post_config,
+        &wrap.randomness,
+        &replicas,
+        wrap.prover_id.clone(),
+    );
+    */
+});

filecoin-proofs/fuzz/fuzz_targets/get_unsealed_range.rs

@@ -0,0 +1,43 @@
+#![no_main]
+use filecoin_proofs::{
+    get_unsealed_range, Commitment, PoRepConfig, PoRepProofPartitions, ProverId, SectorSize, Ticket, UnpaddedByteIndex, UnpaddedBytesAmount, SectorShape2KiB,
+};
+use storage_proofs::sector::SectorId;
+
+use libfuzzer_sys::arbitrary;
+use libfuzzer_sys::fuzz_target;
+use std::path::PathBuf;
+
+#[derive(arbitrary::Arbitrary, Debug)]
+pub struct Wrapper {
+    cache_path: PathBuf,
+    sealed_path: PathBuf,
+    output_path: PathBuf,
+    prover_id: ProverId,
+    sector_id: SectorId,
+    comm_d: Commitment,
+    ticket: Ticket,
+    offset: UnpaddedByteIndex,
+    num_bytes: UnpaddedBytesAmount,
+}
+
+fuzz_target!(|wrap: Wrapper| {
+    let po_rep_config = PoRepConfig {
+        sector_size: SectorSize(2048u64),
+        partitions: PoRepProofPartitions(1),
+    };
+
+    //println!("{:?}\n", wrap);
+    let _ = get_unsealed_range::<_, SectorShape2KiB>(
+        po_rep_config,
+        &wrap.cache_path,
+        &wrap.sealed_path,
+        &wrap.output_path,
+        wrap.prover_id,
+        wrap.sector_id,
+        wrap.comm_d,
+        wrap.ticket,
+        wrap.offset,
+        wrap.num_bytes,
+    );
+});

filecoin-proofs/fuzz/fuzz_targets/seal_commit_phase1.rs

@@ -0,0 +1,41 @@
+#![no_main]
+use filecoin_proofs::{
+    seal_commit_phase1, PieceInfo, PoRepConfig, PoRepProofPartitions, ProverId, SectorSize, Ticket, SealPreCommitOutput,
+};
+use filecoin_proofs::constants::*;
+use storage_proofs::sector::SectorId;
+use libfuzzer_sys::arbitrary;
+use libfuzzer_sys::fuzz_target;
+use std::path::PathBuf;
+
+#[derive(arbitrary::Arbitrary, Debug)]
+pub struct Wrapper {
+    cache_path: PathBuf,
+    replica_path: PathBuf,
+    prover_id: ProverId,
+    sector_id: SectorId,
+    ticket: Ticket,
+    seed: Ticket,
+    pre_commit: SealPreCommitOutput,
+    piece_infos: Vec<PieceInfo>,
+}
+
+fuzz_target!(|wrap: Wrapper| {
+    let po_rep_config = PoRepConfig {
+        sector_size: SectorSize(2048u64),
+        partitions: PoRepProofPartitions(1),
+    };
+
+    //println!("{:?}\n", wrap);
+    let _ = seal_commit_phase1::<_, SectorShape2KiB>(
+        po_rep_config,
+        &wrap.cache_path,
+        &wrap.replica_path,
+        wrap.prover_id.clone(),
+        wrap.sector_id,
+        wrap.ticket,
+        wrap.seed,
+        wrap.pre_commit,
+        &wrap.piece_infos,
+    );
+});

filecoin-proofs/fuzz/fuzz_targets/seal_commit_phase2.rs

@@ -0,0 +1,40 @@
+#![no_main]
+use filecoin_proofs::{
+    seal_commit_phase2, Commitment, DefaultTreeHasher, Ticket, PoRepConfig, PoRepProofPartitions, ProverId, SectorSize, SealCommitPhase1Output, VanillaSealProof
+};
+use filecoin_proofs::constants::*;
+use libfuzzer_sys::arbitrary;
+use libfuzzer_sys::fuzz_target;
+use storage_proofs::hasher::Hasher;
+use storage_proofs::merkle::MerkleProof;
+use storage_proofs::porep::stacked::{generate_replica_id, ColumnProof, EncodingProof, LabelingProof, ReplicaColumnProof};
+use storage_proofs::sector::SectorId;
+use typenum::{U0, U2};
+
+#[derive(arbitrary::Arbitrary, Debug)]
+pub struct Wrapper {
+    //phase1_output: SealCommitPhase1Output,
+    comm_r: Commitment,
+    comm_d: Commitment,
+    seed: Ticket,
+    ticket: Ticket,
+    prover_id: ProverId,
+    sector_id: SectorId,
+}
+
+fuzz_target!(|wrap: Wrapper| {
+    let po_rep_config = PoRepConfig {
+        sector_size: SectorSize(2048u64),
+        partitions: PoRepProofPartitions(1),
+    };
+
+    //println!("{:?}\n", wrap);
+    /*
+    let _ = seal_commit_phase2::<SectorShape2KiB>(
+        po_rep_config,
+        wrap.phase1_output,
+        wrap.prover_id,
+        wrap.sector_id,
+    );
+    */
+});

filecoin-proofs/fuzz/fuzz_targets/seal_pre_commit_phase1.rs

@@ -0,0 +1,39 @@
+#![no_main]
+use filecoin_proofs::{
+    seal_pre_commit_phase1, PieceInfo, PoRepConfig, PoRepProofPartitions, ProverId, SectorSize, Ticket,
+};
+use filecoin_proofs::constants::*;
+use libfuzzer_sys::arbitrary;
+use libfuzzer_sys::fuzz_target;
+use storage_proofs::sector::SectorId;
+use std::path::PathBuf;
+
+#[derive(arbitrary::Arbitrary, Debug)]
+pub struct Wrapper {
+    cache_path: PathBuf,
+    in_path: PathBuf,
+    out_path: PathBuf,
+    prover_id: ProverId,
+    sector_id: SectorId,
+    ticket: Ticket,
+    piece_infos: Vec<PieceInfo>,
+}
+
+fuzz_target!(|wrap: Wrapper| {
+    let po_rep_config = PoRepConfig {
+        sector_size: SectorSize(2048u64),
+        partitions: PoRepProofPartitions(1),
+    };
+
+    //println!("{:?}\n", wrap);
+    let _ = seal_pre_commit_phase1::<_, _, _, SectorShape2KiB>(
+        po_rep_config,
+        &wrap.cache_path,
+        &wrap.in_path,
+        &wrap.out_path,
+        wrap.prover_id.clone(),
+        wrap.sector_id,
+        wrap.ticket,
+        &wrap.piece_infos,
+    );
+});