SealProofVariant and lotus-bench NI-PoRep additions

chain/actors/builtin/miner/utils.go

@@ -8,7 +8,16 @@ import (
 	"github.com/filecoin-project/go-state-types/network"
 )
 
+type SealProofVariant int
+
+const (
+	SealProofVariant_Standard SealProofVariant = iota
+	SealProofVariant_Synthetic
+	SealProofVariant_NonInteractive
+)
+
 var MinSyntheticPoRepVersion = network.Version21
+var MinNonInteractivePoRepVersion = network.Version23
 
 func AllPartSectors(mas State, sget func(Partition) (bitfield.BitField, error)) (bitfield.BitField, error) {
 	var parts []bitfield.BitField
@@ -33,7 +42,18 @@ func AllPartSectors(mas State, sget func(Partition) (bitfield.BitField, error))
 
 // SealProofTypeFromSectorSize returns preferred seal proof type for creating
 // new miner actors and new sectors
-func SealProofTypeFromSectorSize(ssize abi.SectorSize, nv network.Version, synthetic bool) (abi.RegisteredSealProof, error) {
+func SealProofTypeFromSectorSize(ssize abi.SectorSize, nv network.Version, variant SealProofVariant) (abi.RegisteredSealProof, error) {
+	switch variant {
+	case SealProofVariant_Synthetic:
+		if nv < MinSyntheticPoRepVersion {
+			return 0, xerrors.Errorf("synthetic proofs are not supported on network version %d", nv)
+		}
+	case SealProofVariant_NonInteractive:
+		if nv < MinNonInteractivePoRepVersion {
+			return 0, xerrors.Errorf("non-interactive proofs are not supported on network version %d", nv)
+		}
+	}
+
 	switch {
 	case nv < network.Version7:
 		switch ssize {
@@ -67,11 +87,13 @@ func SealProofTypeFromSectorSize(ssize abi.SectorSize, nv network.Version, synth
 			return 0, xerrors.Errorf("unsupported sector size for miner: %v", ssize)
 		}
 
-		if nv >= MinSyntheticPoRepVersion && synthetic {
+		switch variant {
+		case SealProofVariant_Synthetic:
 			return toSynthetic(v)
-		} else {
-			return v, nil
+		case SealProofVariant_NonInteractive:
+			return toNonInteractive(v)
 		}
+		return v, nil
 	}
 
 	return 0, xerrors.Errorf("unsupported network version")
@@ -94,6 +116,23 @@ func toSynthetic(in abi.RegisteredSealProof) (abi.RegisteredSealProof, error) {
 	}
 }
 
+func toNonInteractive(in abi.RegisteredSealProof) (abi.RegisteredSealProof, error) {
+	switch in {
+	case abi.RegisteredSealProof_StackedDrg2KiBV1_1:
+		return abi.RegisteredSealProof_StackedDrg2KiBV1_2_Feat_NiPoRep, nil
+	case abi.RegisteredSealProof_StackedDrg8MiBV1_1:
+		return abi.RegisteredSealProof_StackedDrg8MiBV1_2_Feat_NiPoRep, nil
+	case abi.RegisteredSealProof_StackedDrg512MiBV1_1:
+		return abi.RegisteredSealProof_StackedDrg512MiBV1_2_Feat_NiPoRep, nil
+	case abi.RegisteredSealProof_StackedDrg32GiBV1_1:
+		return abi.RegisteredSealProof_StackedDrg32GiBV1_2_Feat_NiPoRep, nil
+	case abi.RegisteredSealProof_StackedDrg64GiBV1_1:
+		return abi.RegisteredSealProof_StackedDrg64GiBV1_2_Feat_NiPoRep, nil
+	default:
+		return 0, xerrors.Errorf("unsupported conversion to non-interactive: %v", in)
+	}
+}
+
 // WindowPoStProofTypeFromSectorSize returns preferred post proof type for creating
 // new miner actors and new sectors
 func WindowPoStProofTypeFromSectorSize(ssize abi.SectorSize, nv network.Version) (abi.RegisteredPoStProof, error) {

chain/gen/genesis/miners.go

@@ -136,7 +136,11 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sys vm.Syscal
 		i := i
 		m := m
 
-		spt, err := miner.SealProofTypeFromSectorSize(m.SectorSize, nv, synthetic)
+		variant := miner.SealProofVariant_Standard
+		if synthetic {
+			variant = miner.SealProofVariant_Synthetic
+		}
+		spt, err := miner.SealProofTypeFromSectorSize(m.SectorSize, nv, variant)
 		if err != nil {
 			return cid.Undef, err
 		}

cmd/lotus-bench/bench-sectors.sh

@@ -0,0 +1,89 @@
+#!/usr/bin/env bash
+
+# This is an example of how a full sector lifecycle can be benchmarked using `lotus-bench`. The
+# script generates an unsealed sector, runs PC1, PC2, C1, and C2, and prints the duration of each
+# step. The script also prints the proof length and total duration of the lifecycle.
+#
+# Change `flags` to `--non-interactive` to run NI-PoRep, and switch `sector_size` to the desired
+# sector size. The script assumes that the `lotus-bench` binary is in the same directory as the
+# script.
+#
+# Note that for larger sector sizes, /tmp may not have enough space for the full lifecycle.
+
+set -e
+set -o pipefail
+
+tmpdir=/tmp
+
+flags=""
+# flags="--non-interactive"
+sector_size=2KiB
+# sector_size=8MiB
+# sector_size=512MiB
+# sector_size=32GiB
+# sector_size=64GiB
+
+unsealed_file=${tmpdir}/unsealed${sector_size}
+sealed_file=${tmpdir}/sealed${sector_size}
+cache_dir=${tmpdir}/cache${sector_size}
+c1_file=${tmpdir}/c1_${sector_size}.json
+proof_out=${tmpdir}/proof_${sector_size}.hex
+rm -rf $unsealed_file $sealed_file $cache_dir $c1_file
+
+echo "Generating unsealed sector ..."
+read -r unsealed_cid unsealed_size <<< $(./lotus-bench simple addpiece --sector-size $sector_size /dev/zero $unsealed_file | tail -1)
+if [ $? -ne 0 ]; then exit 1; fi
+echo "Unsealed CID: $unsealed_cid"
+echo "Unsealed Size: $unsealed_size"
+
+start_total=$(date +%s%3N)
+
+echo "Running PC1 ..."
+echo "./lotus-bench simple precommit1 --sector-size $sector_size $flags $unsealed_file $sealed_file $cache_dir $unsealed_cid $unsealed_size"
+start_pc1=$(date +%s%3N)
+pc1_output=$(./lotus-bench simple precommit1 --sector-size $sector_size $flags $unsealed_file $sealed_file $cache_dir $unsealed_cid $unsealed_size | tail -1)
+if [ $? -ne 0 ]; then exit 1; fi
+end_pc1=$(date +%s%3N)
+pc1_duration=$((end_pc1 - start_pc1))
+
+echo "Running PC2 ..."
+echo "./lotus-bench simple precommit2 --sector-size $sector_size $flags $sealed_file $cache_dir $pc1_output"
+start_pc2=$(date +%s%3N)
+read -r commd commr <<< $(./lotus-bench simple precommit2 --sector-size $sector_size $flags $sealed_file $cache_dir $pc1_output | tail -1 | sed -E 's/[dr]://g')
+if [ $? -ne 0 ]; then exit 1; fi
+end_pc2=$(date +%s%3N)
+pc2_duration=$((end_pc2 - start_pc2))
+
+echo "CommD CID: $commd"
+echo "CommR CID: $commr"
+
+echo "Running C1 ..."
+echo "./lotus-bench simple commit1 --sector-size $sector_size $flags $sealed_file $cache_dir ${commd} ${commr} $c1_file"
+start_c1=$(date +%s%3N)
+./lotus-bench simple commit1 --sector-size $sector_size $flags $sealed_file $cache_dir ${commd} ${commr} $c1_file
+end_c1=$(date +%s%3N)
+c1_duration=$((end_c1 - start_c1))
+
+echo "Running C2 ..."
+echo "./lotus-bench simple commit2 $flags $c1_file"
+start_c2=$(date +%s%3N)
+proof=$(./lotus-bench simple commit2 $flags $c1_file | tail -1 | sed 's/^proof: //')
+if [ $? -ne 0 ]; then exit 1; fi
+end_c2=$(date +%s%3N)
+c2_duration=$((end_c2 - start_c2))
+
+echo $proof > $proof_out
+echo "Wrote proof to $proof_out"
+
+# $proof is hex, calculate the length of it in bytes
+proof_len=$(echo "scale=0; ${#proof}/2" | bc)
+echo "Proof length: $proof_len"
+
+end_total=$(date +%s%3N)
+total_duration=$((end_total - start_total))
+
+echo "PC1 duration: $((pc1_duration / 1000)).$((pc1_duration % 1000)) seconds"
+echo "PC2 duration: $((pc2_duration / 1000)).$((pc2_duration % 1000)) seconds"
+echo "C1 duration: $((c1_duration / 1000)).$((c1_duration % 1000)) seconds"
+echo "C2 duration: $((c2_duration / 1000)).$((c2_duration % 1000)) seconds"
+echo "Total duration: $((total_duration / 1000)).$((total_duration % 1000)) seconds"

cmd/lotus-bench/main.go

@@ -338,7 +338,7 @@ var sealBenchCmd = &cli.Command{
 
 		if !skipc2 {
 			log.Info("generating winning post candidates")
-			wipt, err := spt(sectorSize, false).RegisteredWinningPoStProof()
+			wipt, err := spt(sectorSize, miner.SealProofVariant_Standard).RegisteredWinningPoStProof()
 			if err != nil {
 				return err
 			}
@@ -556,7 +556,7 @@ func runSeals(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, numSectors int, par
 				Miner:  mid,
 				Number: i,
 			},
-			ProofType: spt(sectorSize, false),
+			ProofType: spt(sectorSize, miner.SealProofVariant_Standard),
 		}
 
 		start := time.Now()
@@ -586,7 +586,7 @@ func runSeals(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, numSectors int, par
 							Miner:  mid,
 							Number: i,
 						},
-						ProofType: spt(sectorSize, false),
+						ProofType: spt(sectorSize, miner.SealProofVariant_Standard),
 					}
 
 					start := time.Now()
@@ -797,7 +797,7 @@ var proveCmd = &cli.Command{
 				Miner:  abi.ActorID(mid),
 				Number: abi.SectorNumber(c2in.SectorNum),
 			},
-			ProofType: spt(abi.SectorSize(c2in.SectorSize), false),
+			ProofType: spt(abi.SectorSize(c2in.SectorSize), miner.SealProofVariant_Standard),
 		}
 
 		fmt.Printf("----\nstart proof computation\n")
@@ -828,8 +828,8 @@ func bps(sectorSize abi.SectorSize, sectorNum int, d time.Duration) string {
 	return types.SizeStr(types.BigInt{Int: bps}) + "/s"
 }
 
-func spt(ssize abi.SectorSize, synth bool) abi.RegisteredSealProof {
-	spt, err := miner.SealProofTypeFromSectorSize(ssize, build.TestNetworkVersion, synth)
+func spt(ssize abi.SectorSize, variant miner.SealProofVariant) abi.RegisteredSealProof {
+	spt, err := miner.SealProofTypeFromSectorSize(ssize, build.TestNetworkVersion, variant)
 	if err != nil {
 		panic(err)
 	}