Fixes for lotus-provider, and conveniences

.circleci/config.yml

@@ -1030,7 +1030,7 @@ workflows:
           requires:
             - build
           suite: utest-unit-rest
-          target: "./blockstore/... ./build/... ./chain/... ./conformance/... ./gateway/... ./journal/... ./lib/... ./markets/... ./paychmgr/... ./provider/... ./tools/..."
+          target: "./blockstore/... ./build/... ./chain/... ./conformance/... ./gateway/... ./journal/... ./lib/... ./markets/... ./paychmgr/... ./tools/..."
 
       - test:
           name: test-unit-storage

Makefile

@@ -128,7 +128,7 @@ an existing lotus binary in your PATH. This may cause problems if you don't run
 
 .PHONY: build
 
-install: install-daemon install-miner install-worker
+install: install-daemon install-miner install-worker install-provider
 
 install-daemon:
 	install -C ./lotus /usr/local/bin/lotus

build/bootstrap/butterflynet.pi

@@ -1,2 +1,2 @@
-/dns4/bootstrap-0.butterfly.fildev.network/tcp/1347/p2p/12D3KooWNwAkUtWuLtKCyyFP2vBzmpTHSrQao7KQx7Xfa8YvSg1N
-/dns4/bootstrap-1.butterfly.fildev.network/tcp/1347/p2p/12D3KooWPn8BDeNcctAcAGuxxiic8uMw2pAi3G5vgdFtfgRs5zBu
+/dns4/bootstrap-0.butterfly.fildev.network/tcp/1347/p2p/12D3KooWRaoPgwJuZdPSN4A2iTeh8xzkZGCEBxan9vMkidHisUgn
+/dns4/bootstrap-1.butterfly.fildev.network/tcp/1347/p2p/12D3KooWMjLCZeEf3VzSWvQYuhe9VzCcrN6RENX9FgmQqiJfQDWs

cli/net.go

@@ -847,7 +847,8 @@ var NetStatCmd = &cli.Command{
 			})
 
 			for _, stat := range stats {
-				printScope(&stat.stat, name+stat.name)
+				tmp := stat.stat
+				printScope(&tmp, name+stat.name)
 			}
 
 		}

cmd/lotus-miner/proving.go

@@ -559,7 +559,8 @@ var provingCheckProvableCmd = &cli.Command{
 		for parIdx, par := range partitions {
 			sectors := make(map[abi.SectorNumber]struct{})
 
-			sectorInfos, err := api.StateMinerSectors(ctx, addr, &par.LiveSectors, types.EmptyTSK)
+			tmp := par.LiveSectors
+			sectorInfos, err := api.StateMinerSectors(ctx, addr, &tmp, types.EmptyTSK)
 			if err != nil {
 				return err
 			}

cmd/lotus-provider/config.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/BurntSushi/toml"
 	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
 	"github.com/filecoin-project/lotus/node/config"
@@ -42,19 +43,26 @@ var configDefaultCmd = &cli.Command{
 		},
 	},
 	Action: func(cctx *cli.Context) error {
-		c := config.DefaultLotusProvider()
-
-		cb, err := config.ConfigUpdate(c, nil, config.Commented(!cctx.Bool("no-comment")), config.DefaultKeepUncommented(), config.NoEnv())
+		comment := !cctx.Bool("no-comment")
+		cfg, err := getDefaultConfig(comment)
 		if err != nil {
 			return err
 		}
-
-		fmt.Print(string(cb))
+		fmt.Print(cfg)
 
 		return nil
 	},
 }
 
+func getDefaultConfig(comment bool) (string, error) {
+	c := config.DefaultLotusProvider()
+	cb, err := config.ConfigUpdate(c, nil, config.Commented(comment), config.DefaultKeepUncommented(), config.NoEnv())
+	if err != nil {
+		return "", err
+	}
+	return string(cb), nil
+}
+
 var configSetCmd = &cli.Command{
 	Name:      "set",
 	Aliases:   []string{"add"},
@@ -190,7 +198,7 @@ var configRmCmd = &cli.Command{
 var configViewCmd = &cli.Command{
 	Name:      "interpret",
 	Aliases:   []string{"view", "stacked", "stack"},
-	Usage:     "Interpret stacked config layers by this version of lotus-provider.",
+	Usage:     "Interpret stacked config layers by this version of lotus-provider, with system-generated comments.",
 	ArgsUsage: "a list of layers to be interpreted as the final config",
 	Flags: []cli.Flag{
 		&cli.StringSliceFlag{
@@ -209,10 +217,12 @@ var configViewCmd = &cli.Command{
 		if err != nil {
 			return err
 		}
-
-		e := toml.NewEncoder(os.Stdout)
-		e.Indent = "  "
-		return e.Encode(lp)
+		cb, err := config.ConfigUpdate(lp, config.DefaultLotusProvider(), config.Commented(true), config.DefaultKeepUncommented(), config.NoEnv())
+		if err != nil {
+			return xerrors.Errorf("cannot interpret config: %w", err)
+		}
+		fmt.Println(string(cb))
+		return nil
 	},
 }
 

cmd/lotus-provider/main.go

@@ -43,6 +43,7 @@ func main() {
 		runCmd,
 		stopCmd,
 		configCmd,
+		provingCmd,
 		//backupCmd,
 		//lcli.WithCategory("chain", actorCmd),
 		//lcli.WithCategory("storage", sectorsCmd),

cmd/lotus-provider/migrate.go

@@ -161,7 +161,11 @@ environment variable LOTUS_WORKER_WINDOWPOST.
 	}
 
 	if !lo.Contains(titles, "base") {
-		_, err = db.Exec(ctx, "INSERT INTO harmony_config (title, config) VALUES ('base', '')", "base")
+		cfg, err := getDefaultConfig(true)
+		if err != nil {
+			return xerrors.Errorf("Cannot get default config: %w", err)
+		}
+		_, err = db.Exec(ctx, "INSERT INTO harmony_config (title, config) VALUES ('base', '$1')", cfg)
 		if err != nil {
 			return err
 		}

cmd/lotus-provider/proving.go

@@ -0,0 +1,180 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/dline"
+
+	"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
+	"github.com/filecoin-project/lotus/provider"
+)
+
+var provingCmd = &cli.Command{
+	Name:  "proving",
+	Usage: "Utility functions for proving sectors",
+	Subcommands: []*cli.Command{
+		//provingInfoCmd,
+		provingCompute,
+	},
+}
+
+var provingCompute = &cli.Command{
+	Name:  "compute",
+	Usage: "Compute a proof-of-spacetime for a sector (requires the sector to be pre-sealed)",
+	Subcommands: []*cli.Command{
+		provingComputeWindowPoStCmd,
+		scheduleWindowPostCmd,
+	},
+}
+
+var scheduleWindowPostCmd = &cli.Command{
+	Name:  "test-window-post",
+	Usage: "FOR TESTING: a way to test the windowpost scheduler. Use with 1 lotus-provider running only",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name: "i_have_set_LOTUS_PROVDER_NO_SEND_to_true",
+		},
+		&cli.Uint64Flag{
+			Name:  "deadline",
+			Usage: "deadline to compute WindowPoSt for ",
+			Value: 0,
+		},
+		&cli.StringSliceFlag{
+			Name:  "layers",
+			Usage: "list of layers to be interpreted (atop defaults). Default: base",
+			Value: cli.NewStringSlice("base"),
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		ctx := context.Background()
+
+		if !cctx.Bool("i_have_set_LOTUS_PROVDER_NO_SEND_to_true") {
+			log.Info("This command is for testing only. It will not send any messages to the chain. If you want to run it, set LOTUS_PROVDER_NO_SEND=true on the lotus-provider environment.")
+			return nil
+		}
+		deps, err := getDeps(ctx, cctx)
+		if err != nil {
+			return err
+		}
+
+		ts, err := deps.full.ChainHead(ctx)
+		if err != nil {
+			return xerrors.Errorf("cannot get chainhead %w", err)
+		}
+		ht := ts.Height()
+
+		addr, err := address.NewFromString(deps.cfg.Addresses.MinerAddresses[0])
+		if err != nil {
+			return xerrors.Errorf("cannot get miner address %w", err)
+		}
+		maddr, err := address.IDFromAddress(addr)
+		if err != nil {
+			return xerrors.Errorf("cannot get miner id %w", err)
+		}
+		did, err := deps.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) {
+			_, err = tx.Exec(`INSERT INTO harmony_task (name, posted_time, added_by) VALUES ('WdPost', CURRENT_TIMESTAMP, 123)`)
+			if err != nil {
+				log.Error("inserting harmony_task: ", err)
+				return false, xerrors.Errorf("inserting harmony_task: %w", err)
+			}
+			var id int64
+			if err = tx.QueryRow(`SELECT id FROM harmony_task ORDER BY update_time DESC LIMIT 1`).Scan(&id); err != nil {
+				log.Error("getting inserted id: ", err)
+				return false, xerrors.Errorf("getting inserted id: %w", err)
+			}
+			_, err = tx.Exec(`INSERT INTO wdpost_partition_tasks 
+			(task_id, sp_id, proving_period_start, deadline_index, partition_index) VALUES ($1, $2, $3, $4, $5)`,
+				id, maddr, ht, cctx.Uint64("deadline"), 0)
+			if err != nil {
+				log.Error("inserting wdpost_partition_tasks: ", err)
+				return false, xerrors.Errorf("inserting wdpost_partition_tasks: %w", err)
+			}
+			return true, nil
+		})
+		if err != nil {
+			return xerrors.Errorf("writing SQL transaction: %w", err)
+		}
+		log.Infof("Inserted task %v", did)
+		return nil
+	},
+}
+
+var provingComputeWindowPoStCmd = &cli.Command{
+	Name:    "windowed-post",
+	Aliases: []string{"window-post"},
+	Usage:   "Compute WindowPoSt for performance and configuration testing.",
+	Description: `Note: This command is intended to be used to verify PoSt compute performance.
+It will not send any messages to the chain. Since it can compute any deadline, output may be incorrectly timed for the chain.`,
+	ArgsUsage: "[deadline index]",
+	Flags: []cli.Flag{
+		&cli.Uint64Flag{
+			Name:  "deadline",
+			Usage: "deadline to compute WindowPoSt for ",
+			Value: 0,
+		},
+		&cli.StringSliceFlag{
+			Name:  "layers",
+			Usage: "list of layers to be interpreted (atop defaults). Default: base",
+			Value: cli.NewStringSlice("base"),
+		},
+		&cli.StringFlag{
+			Name:  "storage-json",
+			Usage: "path to json file containing storage config",
+			Value: "~/.lotus-provider/storage.json",
+		},
+		&cli.Uint64Flag{
+			Name:  "partition",
+			Usage: "partition to compute WindowPoSt for",
+			Value: 0,
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+
+		ctx := context.Background()
+		deps, err := getDeps(ctx, cctx)
+		if err != nil {
+			return err
+		}
+
+		wdPostTask, wdPoStSubmitTask, derlareRecoverTask, err := provider.WindowPostScheduler(ctx, deps.cfg.Fees, deps.cfg.Proving, deps.full, deps.verif, deps.lw,
+			deps.as, deps.maddrs, deps.db, deps.stor, deps.si, deps.cfg.Subsystems.WindowPostMaxTasks)
+		if err != nil {
+			return err
+		}
+		_, _ = wdPoStSubmitTask, derlareRecoverTask
+
+		if len(deps.maddrs) == 0 {
+			return errors.New("no miners to compute WindowPoSt for")
+		}
+		head, err := deps.full.ChainHead(ctx)
+		if err != nil {
+			return xerrors.Errorf("failed to get chain head: %w", err)
+		}
+
+		di := dline.NewInfo(head.Height(), cctx.Uint64("deadline"), 0, 0, 0, 10 /*challenge window*/, 0, 0)
+
+		for _, maddr := range deps.maddrs {
+			out, err := wdPostTask.DoPartition(ctx, head, address.Address(maddr), di, cctx.Uint64("partition"))
+			if err != nil {
+				fmt.Println("Error computing WindowPoSt for miner", maddr, err)
+				continue
+			}
+			fmt.Println("Computed WindowPoSt for miner", maddr, ":")
+			err = json.NewEncoder(os.Stdout).Encode(out)
+			if err != nil {
+				fmt.Println("Could not encode WindowPoSt output for miner", maddr, err)
+				continue
+			}
+		}
+
+		return nil
+	},
+}