From be384129012414f2b6cf7df29cb884d5136c1577 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Fri, 3 Apr 2020 18:17:57 -0400 Subject: [PATCH 1/2] Add an API method to calculate piece CIDs --- api/api_full.go | 7 ++ api/apistruct/struct.go | 9 ++ cli/client.go | 66 +++++++++++++- node/impl/client/client.go | 179 ++++++++++++++++++++++++++----------- 4 files changed, 210 insertions(+), 51 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 724b721d754..3bd9434f6d1 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -101,6 +101,8 @@ type FullNode interface { ClientFindData(ctx context.Context, root cid.Cid) ([]QueryOffer, error) ClientRetrieve(ctx context.Context, order RetrievalOrder, ref FileRef) error ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error) + ClientCalcCommP(ctx context.Context, inpath string, miner address.Address) (*CommPRet, error) + ClientGenCar(ctx context.Context, ref FileRef, outpath string) error // ClientUnimport removes references to the specified file from filestore //ClientUnimport(path string) @@ -398,3 +400,8 @@ type BlockTemplate struct { Epoch abi.ChainEpoch Timestamp uint64 } + +type CommPRet struct { + Root cid.Cid + Size abi.UnpaddedPieceSize +} diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 3bbe9d40d19..603af0ad0eb 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -112,6 +112,8 @@ type FullNodeStruct struct { ClientListDeals func(ctx context.Context) ([]api.DealInfo, error) `perm:"write"` ClientRetrieve func(ctx context.Context, order api.RetrievalOrder, ref api.FileRef) error `perm:"admin"` ClientQueryAsk func(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error) `perm:"read"` + ClientCalcCommP func(ctx context.Context, inpath string, miner address.Address) (*api.CommPRet, error) `perm:"read"` + ClientGenCar func(ctx context.Context, ref api.FileRef, outpath string) error `perm:"write"` StateNetworkName func(context.Context) (dtypes.NetworkName, error) `perm:"read"` StateMinerSectors func(context.Context, address.Address, types.TipSetKey) ([]*api.ChainSectorInfo, error) `perm:"read"` @@ -309,6 +311,13 @@ func (c *FullNodeStruct) ClientRetrieve(ctx context.Context, order api.Retrieval func (c *FullNodeStruct) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error) { return c.Internal.ClientQueryAsk(ctx, p, miner) } +func (c *FullNodeStruct) ClientCalcCommP(ctx context.Context, inpath string, miner address.Address) (*api.CommPRet, error) { + return c.Internal.ClientCalcCommP(ctx, inpath, miner) +} + +func (c *FullNodeStruct) ClientGenCar(ctx context.Context, ref api.FileRef, outpath string) error { + return c.Internal.ClientGenCar(ctx, ref, outpath) +} func (c *FullNodeStruct) MpoolPending(ctx context.Context, tsk types.TipSetKey) ([]*types.SignedMessage, error) { return c.Internal.MpoolPending(ctx, tsk) diff --git a/cli/client.go b/cli/client.go index 9f6b12e6933..7a5359ce2f3 100644 --- a/cli/client.go +++ b/cli/client.go @@ -25,12 +25,14 @@ var clientCmd = &cli.Command{ Usage: "Make deals, store data, retrieve data", Subcommands: []*cli.Command{ clientImportCmd, + clientCommPCmd, clientLocalCmd, clientDealCmd, clientFindCmd, clientRetrieveCmd, clientQueryAskCmd, clientListDeals, + clientCarGenCmd, }, } @@ -41,7 +43,7 @@ var clientImportCmd = &cli.Command{ Flags: []cli.Flag{ &cli.BoolFlag{ Name: "car", - Usage: "export to a car file instead of a regular file", + Usage: "import from a car file instead of a regular file", }, }, Action: func(cctx *cli.Context) error { @@ -69,6 +71,68 @@ var clientImportCmd = &cli.Command{ }, } +var clientCommPCmd = &cli.Command{ + Name: "commP", + Usage: "calculate the piece-cid (commP) of a CAR file", + ArgsUsage: "[inputFile minerAddress]", + Action: func(cctx *cli.Context) error { + api, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + ctx := ReqContext(cctx) + + if cctx.Args().Len() != 2 { + return fmt.Errorf("usage: commP ") + } + + miner, err := address.NewFromString(cctx.Args().Get(1)) + if err != nil { + return err + } + + ret, err := api.ClientCalcCommP(ctx, cctx.Args().Get(0), miner) + + if err != nil { + return err + } + fmt.Println("CID: ", ret.Root) + fmt.Println("Piece size: ", ret.Size) + return nil + }, +} + +var clientCarGenCmd = &cli.Command{ + Name: "generate-car", + Usage: "generate a car file from input", + ArgsUsage: "[inputPath outputPath]", + Action: func(cctx *cli.Context) error { + api, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + ctx := ReqContext(cctx) + + if cctx.Args().Len() != 2 { + return fmt.Errorf("usage: generate-car ") + } + + ref := lapi.FileRef{ + Path: cctx.Args().First(), + IsCAR: false, + } + + op := cctx.Args().Get(1) + + if err = api.ClientGenCar(ctx, ref, op); err != nil { + return err + } + return nil + }, +} + var clientLocalCmd = &cli.Command{ Name: "local", Usage: "List locally imported data", diff --git a/node/impl/client/client.go b/node/impl/client/client.go index b3249b91c84..de2e3be3c7d 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -3,6 +3,11 @@ package client import ( "context" "errors" + "github.com/filecoin-project/go-fil-markets/pieceio" + ipldfree "github.com/ipld/go-ipld-prime/impl/free" + "github.com/ipld/go-ipld-prime/traversal/selector" + "github.com/ipld/go-ipld-prime/traversal/selector/builder" + "io" "os" @@ -202,63 +207,15 @@ func (a *API) ClientFindData(ctx context.Context, root cid.Cid) ([]api.QueryOffe } func (a *API) ClientImport(ctx context.Context, ref api.FileRef) (cid.Cid, error) { - f, err := os.Open(ref.Path) - if err != nil { - return cid.Undef, err - } - - stat, err := f.Stat() - if err != nil { - return cid.Undef, err - } - - file, err := files.NewReaderPathFile(ref.Path, f, stat) - if err != nil { - return cid.Undef, err - } - if ref.IsCAR { - var store car.Store - if a.Filestore == nil { - store = a.Blockstore - } else { - store = (*filestore.Filestore)(a.Filestore) - } - result, err := car.LoadCar(store, file) - if err != nil { - return cid.Undef, err - } - - if len(result.Roots) != 1 { - return cid.Undef, xerrors.New("cannot import car with more than one root") - } - - return result.Roots[0], nil - } bufferedDS := ipld.NewBufferedDAG(ctx, a.LocalDAG) + nd, err := a.clientImport(ref, bufferedDS) - params := ihelper.DagBuilderParams{ - Maxlinks: build.UnixfsLinksPerLevel, - RawLeaves: true, - CidBuilder: nil, - Dagserv: bufferedDS, - NoCopy: true, - } - - db, err := params.New(chunker.NewSizeSplitter(file, int64(build.UnixfsChunkSize))) - if err != nil { - return cid.Undef, err - } - nd, err := balanced.Layout(db) if err != nil { return cid.Undef, err } - if err := bufferedDS.Commit(); err != nil { - return cid.Undef, err - } - - return nd.Cid(), nil + return nd, nil } func (a *API) ClientImportLocal(ctx context.Context, f io.Reader) (cid.Cid, error) { @@ -410,3 +367,125 @@ func (a *API) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Addre } return signedAsk, nil } + +func (a *API) ClientCalcCommP(ctx context.Context, inpath string, miner address.Address) (*api.CommPRet, error) { + ssize, err := a.StateMinerSectorSize(ctx, miner, types.EmptyTSK) + if err != nil { + return nil, xerrors.Errorf("failed checking miners sector size: %w", err) + } + + rt, _, err := ffiwrapper.ProofTypeFromSectorSize(ssize) + if err != nil { + return nil, xerrors.Errorf("bad sector size: %w", err) + } + + rdr, err := os.Open(inpath) + if err != nil { + return nil, err + } + + stat, err := rdr.Stat() + if err != nil { + return nil, err + } + + commP, pieceSize, err := pieceio.GeneratePieceCommitment(rt, rdr, uint64(stat.Size())) + + if err != nil { + return nil, xerrors.Errorf("computing commP failed: %w", err) + } + + return &api.CommPRet{ + Root: commP, + Size: pieceSize, + }, nil +} + +func (a *API) ClientGenCar(ctx context.Context, ref api.FileRef, outputPath string) error { + + bufferedDS := ipld.NewBufferedDAG(ctx, a.LocalDAG) + c, err := a.clientImport(ref, bufferedDS) + + if err != nil { + return err + } + + defer bufferedDS.Remove(ctx, c) + ssb := builder.NewSelectorSpecBuilder(ipldfree.NodeBuilder()) + + // entire DAG selector + allSelector := ssb.ExploreRecursive(selector.RecursionLimitNone(), + ssb.ExploreAll(ssb.ExploreRecursiveEdge())).Node() + + f, err := os.Create(outputPath) + defer f.Close() + if err != nil { + return err + } + + sc := car.NewSelectiveCar(ctx, a.Blockstore, []car.Dag{{Root: c, Selector: allSelector}}) + if err = sc.Write(f); err != nil { + return err + } + + return nil +} + +func (a *API) clientImport(ref api.FileRef, bufferedDS *ipld.BufferedDAG) (cid.Cid, error) { + f, err := os.Open(ref.Path) + if err != nil { + return cid.Undef, err + } + + stat, err := f.Stat() + if err != nil { + return cid.Undef, err + } + + file, err := files.NewReaderPathFile(ref.Path, f, stat) + if err != nil { + return cid.Undef, err + } + + if ref.IsCAR { + var store car.Store + if a.Filestore == nil { + store = a.Blockstore + } else { + store = (*filestore.Filestore)(a.Filestore) + } + result, err := car.LoadCar(store, file) + if err != nil { + return cid.Undef, err + } + + if len(result.Roots) != 1 { + return cid.Undef, xerrors.New("cannot import car with more than one root") + } + + return result.Roots[0], nil + } + + params := ihelper.DagBuilderParams{ + Maxlinks: build.UnixfsLinksPerLevel, + RawLeaves: true, + CidBuilder: nil, + Dagserv: bufferedDS, + NoCopy: true, + } + + db, err := params.New(chunker.NewSizeSplitter(file, int64(build.UnixfsChunkSize))) + if err != nil { + return cid.Undef, err + } + nd, err := balanced.Layout(db) + if err != nil { + return cid.Undef, err + } + + if err := bufferedDS.Commit(); err != nil { + return cid.Undef, err + } + + return nd.Cid(), nil +} From e5af3f37368e89eb1726c702392ad07599709f04 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Sun, 5 Apr 2020 23:30:11 -0400 Subject: [PATCH 2/2] Treat proposed deals as manual iff a commP is provided --- cli/client.go | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/cli/client.go b/cli/client.go index 7a5359ce2f3..530fd7ae0eb 100644 --- a/cli/client.go +++ b/cli/client.go @@ -160,17 +160,13 @@ var clientDealCmd = &cli.Command{ Usage: "Initialize storage deal with a miner", ArgsUsage: "[dataCid miner price duration]", Flags: []cli.Flag{ - &cli.BoolFlag{ - Name: "manual-transfer", - Usage: "data will be transferred out of band", - }, &cli.StringFlag{ Name: "manual-piece-cid", - Usage: "manually specify piece commitment for data", + Usage: "manually specify piece commitment for data (dataCid must be to a car file)", }, &cli.Int64Flag{ Name: "manual-piece-size", - Usage: "if manually specifying piece cid, used to specify size", + Usage: "if manually specifying piece cid, used to specify size (dataCid must be to a car file)", }, &cli.StringFlag{ Name: "from", @@ -189,7 +185,7 @@ var clientDealCmd = &cli.Command{ return xerrors.New("expected 4 args: dataCid, miner, price, duration") } - // [data, miner, dur] + // [data, miner, price, dur] data, err := cid.Parse(cctx.Args().Get(0)) if err != nil { @@ -230,9 +226,6 @@ var clientDealCmd = &cli.Command{ TransferType: storagemarket.TTGraphsync, Root: data, } - if cctx.Bool("manual-transfer") { - ref.TransferType = storagemarket.TTManual - } if mpc := cctx.String("manual-piece-cid"); mpc != "" { c, err := cid.Parse(mpc) @@ -248,6 +241,8 @@ var clientDealCmd = &cli.Command{ } ref.PieceSize = abi.UnpaddedPieceSize(psize) + + ref.TransferType = storagemarket.TTManual } proposal, err := api.ClientStartDeal(ctx, &lapi.StartDealParams{