From b85b50afbcc8ad24fda710369fbcd3ad4a001c5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 17 Sep 2020 19:57:42 +0200 Subject: [PATCH 1/5] shed: bitfield intersect command --- cmd/lotus-shed/bitfield.go | 113 +++++++++++++++++++++++++++---------- 1 file changed, 83 insertions(+), 30 deletions(-) diff --git a/cmd/lotus-shed/bitfield.go b/cmd/lotus-shed/bitfield.go index 79ce214eee9..78f6f02fecc 100644 --- a/cmd/lotus-shed/bitfield.go +++ b/cmd/lotus-shed/bitfield.go @@ -28,6 +28,7 @@ var bitFieldCmd = &cli.Command{ bitFieldRunsCmd, bitFieldStatCmd, bitFieldDecodeCmd, + bitFieldIntersectCmd, }, } @@ -200,46 +201,98 @@ var bitFieldDecodeCmd = &cli.Command{ }, }, Action: func(cctx *cli.Context) error { - var val string - if cctx.Args().Present() { - val = cctx.Args().Get(0) - } else { - b, err := ioutil.ReadAll(os.Stdin) - if err != nil { - return err - } - val = string(b) + rle, err := decode(cctx, 0) + if err != nil { + return err } - var dec []byte - switch cctx.String("enc") { - case "base64": - d, err := base64.StdEncoding.DecodeString(val) - if err != nil { - return fmt.Errorf("decoding base64 value: %w", err) - } - dec = d - case "hex": - d, err := hex.DecodeString(val) - if err != nil { - return fmt.Errorf("decoding hex value: %w", err) - } - dec = d - default: - return fmt.Errorf("unrecognized encoding: %s", cctx.String("enc")) + vals, err := rle.All(100000000000) + if err != nil { + return xerrors.Errorf("getting all items: %w", err) } + fmt.Println(vals) + + return nil + }, +} - rle, err := bitfield.NewFromBytes(dec) +var bitFieldIntersectCmd = &cli.Command{ + Name: "intersect", + Description: "intersect 2 bitfields and print the resulting bitfield as base64", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "enc", + Value: "base64", + Usage: "specify input encoding to parse", + }, + }, + Action: func(cctx *cli.Context) error { + b, err := decode(cctx, 1) if err != nil { - return xerrors.Errorf("failed to parse bitfield: %w", err) + return err } - vals, err := rle.All(100000000000) + a, err := decode(cctx, 0) if err != nil { - return xerrors.Errorf("getting all items: %w", err) + return err + } + + o, err := bitfield.IntersectBitField(a, b) + if err != nil { + return xerrors.Errorf("intersect: %w", err) + } + + s, err := o.RunIterator() + if err != nil { + return err } - fmt.Println(vals) + + bytes, err := rlepluslazy.EncodeRuns(s, []byte{}) + if err != nil { + return err + } + + fmt.Println(base64.StdEncoding.EncodeToString(bytes)) return nil }, } + +func decode(cctx *cli.Context, a int) (bitfield.BitField, error) { + var val string + if cctx.Args().Present() { + if a >= cctx.NArg() { + return bitfield.BitField{}, xerrors.Errorf("need more than %d args", a) + } + val = cctx.Args().Get(a) + } else { + if a > 0 { + return bitfield.BitField{}, xerrors.Errorf("need more than %d args", a) + } + b, err := ioutil.ReadAll(os.Stdin) + if err != nil { + return bitfield.BitField{}, err + } + val = string(b) + } + + var dec []byte + switch cctx.String("enc") { + case "base64": + d, err := base64.StdEncoding.DecodeString(val) + if err != nil { + return bitfield.BitField{}, fmt.Errorf("decoding base64 value: %w", err) + } + dec = d + case "hex": + d, err := hex.DecodeString(val) + if err != nil { + return bitfield.BitField{}, fmt.Errorf("decoding hex value: %w", err) + } + dec = d + default: + return bitfield.BitField{}, fmt.Errorf("unrecognized encoding: %s", cctx.String("enc")) + } + + return bitfield.NewFromBytes(dec) +} From dcc56434732783a400a679c3adcd59b295586a04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 18 Sep 2020 02:38:30 +0200 Subject: [PATCH 2/5] shed: Even more bitfield utils --- cmd/lotus-shed/bitfield.go | 90 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/cmd/lotus-shed/bitfield.go b/cmd/lotus-shed/bitfield.go index 78f6f02fecc..e1e06573ce5 100644 --- a/cmd/lotus-shed/bitfield.go +++ b/cmd/lotus-shed/bitfield.go @@ -4,6 +4,7 @@ import ( "encoding/base64" "encoding/hex" "fmt" + "io" "io/ioutil" "os" @@ -29,6 +30,8 @@ var bitFieldCmd = &cli.Command{ bitFieldStatCmd, bitFieldDecodeCmd, bitFieldIntersectCmd, + bitFieldEncodeCmd, + bitFieldSubCmd, }, } @@ -258,6 +261,93 @@ var bitFieldIntersectCmd = &cli.Command{ }, } + +var bitFieldSubCmd = &cli.Command{ + Name: "sub", + Description: "subtract 2 bitfields and print the resulting bitfield as base64", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "enc", + Value: "base64", + Usage: "specify input encoding to parse", + }, + }, + Action: func(cctx *cli.Context) error { + b, err := decode(cctx, 1) + if err != nil { + return err + } + + a, err := decode(cctx, 0) + if err != nil { + return err + } + + o, err := bitfield.SubtractBitField(a, b) + if err != nil { + return xerrors.Errorf("intersect: %w", err) + } + + s, err := o.RunIterator() + if err != nil { + return err + } + + bytes, err := rlepluslazy.EncodeRuns(s, []byte{}) + if err != nil { + return err + } + + fmt.Println(base64.StdEncoding.EncodeToString(bytes)) + + return nil + }, +} + +var bitFieldEncodeCmd = &cli.Command{ + Name: "encode", + Description: "encode a series of decimal numbers into a bitfield", + ArgsUsage: "[infile]", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "enc", + Value: "base64", + Usage: "specify input encoding to parse", + }, + }, + Action: func(cctx *cli.Context) error { + f, err := os.Open(cctx.Args().First()) + if err != nil { + return err + } + defer f.Close() + + out := bitfield.New() + for { + var i uint64 + _, err := fmt.Fscan(f, &i) + if err == io.EOF { + break + } + out.Set(i) + } + + s, err := out.RunIterator() + if err != nil { + return err + } + + bytes, err := rlepluslazy.EncodeRuns(s, []byte{}) + if err != nil { + return err + } + + fmt.Println(base64.StdEncoding.EncodeToString(bytes)) + + return nil + }, +} + func decode(cctx *cli.Context, a int) (bitfield.BitField, error) { var val string if cctx.Args().Present() { From 8419dda84b1cbfa194a15a50529a5cc4cf862546 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 18 Sep 2020 02:39:08 +0200 Subject: [PATCH 3/5] wdpost: Only add proven partitions to message params --- storage/wdpost_run.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 17b56744053..532d7144b51 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -431,7 +431,9 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty postSkipped := bitfield.New() var postOut []proof.PoStProof somethingToProve := true + for retries := 0; retries < 5; retries++ { + var partitions []miner.PoStPartition var sinfos []proof.SectorInfo for partIdx, partition := range batch { // TODO: Can do this in parallel @@ -477,7 +479,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty } sinfos = append(sinfos, ssi...) - params.Partitions = append(params.Partitions, miner.PoStPartition{ + partitions = append(params.Partitions, miner.PoStPartition{ Index: uint64(batchPartitionStartIdx + partIdx), Skipped: skipped, }) @@ -511,6 +513,8 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty if err == nil { // Proof generation successful, stop retrying + params.Partitions = append(params.Partitions, partitions...) + break } From 57c1eac37a29a1c39b85e4d245fdd1b0632f5069 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 18 Sep 2020 02:40:01 +0200 Subject: [PATCH 4/5] gofmt --- cmd/lotus-shed/bitfield.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/lotus-shed/bitfield.go b/cmd/lotus-shed/bitfield.go index e1e06573ce5..9cc63228389 100644 --- a/cmd/lotus-shed/bitfield.go +++ b/cmd/lotus-shed/bitfield.go @@ -261,7 +261,6 @@ var bitFieldIntersectCmd = &cli.Command{ }, } - var bitFieldSubCmd = &cli.Command{ Name: "sub", Description: "subtract 2 bitfields and print the resulting bitfield as base64", @@ -307,7 +306,7 @@ var bitFieldSubCmd = &cli.Command{ var bitFieldEncodeCmd = &cli.Command{ Name: "encode", Description: "encode a series of decimal numbers into a bitfield", - ArgsUsage: "[infile]", + ArgsUsage: "[infile]", Flags: []cli.Flag{ &cli.StringFlag{ Name: "enc", From 26bbd9fb9d13207ed6f32d769bfb8ebfb548f795 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 18 Sep 2020 03:00:31 +0200 Subject: [PATCH 5/5] wdpost: append is hard --- cmd/lotus-shed/bitfield.go | 2 +- storage/wdpost_run.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-shed/bitfield.go b/cmd/lotus-shed/bitfield.go index 9cc63228389..442cbef4846 100644 --- a/cmd/lotus-shed/bitfield.go +++ b/cmd/lotus-shed/bitfield.go @@ -319,7 +319,7 @@ var bitFieldEncodeCmd = &cli.Command{ if err != nil { return err } - defer f.Close() + defer f.Close() // nolint out := bitfield.New() for { diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 532d7144b51..463c7a3c509 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -479,7 +479,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty } sinfos = append(sinfos, ssi...) - partitions = append(params.Partitions, miner.PoStPartition{ + partitions = append(partitions, miner.PoStPartition{ Index: uint64(batchPartitionStartIdx + partIdx), Skipped: skipped, })