feat:enable pad car file to become a piece file #35

.gitignore

@@ -18,3 +18,4 @@
 # vendor/
 
 .idea
+.vscode

chunk.go

@@ -22,15 +22,16 @@ type GraphBuildCallback interface {
 }
 
 type commPCallback struct {
-	carDir string
-	rename bool
+	carDir     string
+	rename     bool
+	addPadding bool
 }
 
 func (cc *commPCallback) OnSuccess(node ipld.Node, graphName, fsDetail string) {
 	fmt.Println("xxxxx")
 	commpStartTime := time.Now()
 	carfilepath := path.Join(cc.carDir, node.Cid().String()+".car")
-	cpRes, err := CalcCommP(context.TODO(), carfilepath, cc.rename)
+	cpRes, err := CalcCommP(context.TODO(), carfilepath, cc.rename, cc.addPadding)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -56,7 +57,7 @@ func (cc *commPCallback) OnSuccess(node ipld.Node, graphName, fsDetail string) {
 	defer csvWriter.Flush()
 	if isCreateAction {
 		csvWriter.Write([]string{
-			"playload_cid","filename","piece_cid","payload_size","piece_size","detail",
+			"playload_cid", "filename", "piece_cid", "payload_size", "piece_size", "detail",
 		})
 	}
 
@@ -112,8 +113,8 @@ func (cc *errCallback) OnError(err error) {
 	log.Fatal(err)
 }
 
-func CommPCallback(carDir string, rename bool) GraphBuildCallback {
-	return &commPCallback{carDir: carDir, rename: rename}
+func CommPCallback(carDir string, rename, addPadding bool) GraphBuildCallback {
+	return &commPCallback{carDir: carDir, rename: rename, addPadding: addPadding}
 }
 
 func CSVCallback(carDir string) GraphBuildCallback {

cmd/graphsplit/main.go

@@ -75,10 +75,15 @@ var chunkCmd = &cli.Command{
 			Usage: "create a mainfest.csv in car-dir to save mapping of data-cids, slice names, piece-cids and piece-sizes",
 		},
 		&cli.BoolFlag{
-			Name: "rename",
+			Name:  "rename",
 			Value: false,
 			Usage: "rename carfile to piece",
 		},
+		&cli.BoolFlag{
+			Name:  "add-padding",
+			Value: false,
+			Usage: "add padding to carfile in order to convert it to piece file",
+		},
 	},
 	Action: func(c *cli.Context) error {
 		ctx := context.Background()
@@ -97,7 +102,7 @@ var chunkCmd = &cli.Command{
 		targetPath := c.Args().First()
 		var cb graphsplit.GraphBuildCallback
 		if c.Bool("calc-commp") {
-			cb = graphsplit.CommPCallback(carDir, c.Bool("rename"))
+			cb = graphsplit.CommPCallback(carDir, c.Bool("rename"), c.Bool("add-padding"))
 		} else if c.Bool("save-manifest") {
 			cb = graphsplit.CSVCallback(carDir)
 		} else {
@@ -146,12 +151,23 @@ var restoreCmd = &cli.Command{
 var commpCmd = &cli.Command{
 	Name:  "commP",
 	Usage: "PieceCID and PieceSize calculation",
-	Flags: []cli.Flag{},
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:  "rename",
+			Value: false,
+			Usage: "rename carfile to piece",
+		},
+		&cli.BoolFlag{
+			Name:  "add-padding",
+			Value: false,
+			Usage: "add padding to carfile in order to convert it to piece file",
+		},
+	},
 	Action: func(c *cli.Context) error {
 		ctx := context.Background()
 		targetPath := c.Args().First()
 
-		res, err := graphsplit.CalcCommP(ctx, targetPath, false)
+		res, err := graphsplit.CalcCommP(ctx, targetPath, c.Bool("rename"), c.Bool("add-padding"))
 		if err != nil {
 			return err
 		}

commp.go

@@ -11,19 +11,20 @@ import (
 	"github.com/filecoin-project/go-commp-utils/ffiwrapper"
 	"github.com/filecoin-project/go-padreader"
 	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filedrive-team/filehelper/carv1"
 	"github.com/ipfs/go-cid"
 	"github.com/ipld/go-car"
 	"golang.org/x/xerrors"
 )
 
 type CommPRet struct {
-	Root cid.Cid
+	Root        cid.Cid
 	PayloadSize int64
-	Size abi.UnpaddedPieceSize
+	Size        abi.UnpaddedPieceSize
 }
 
 // almost copy paste from https://github.com/filecoin-project/lotus/node/impl/client/client.go#L749-L770
-func CalcCommP(ctx context.Context, inpath string, rename bool) (*CommPRet, error) {
+func CalcCommP(ctx context.Context, inpath string, rename, addPadding bool) (*CommPRet, error) {
 	dir, _ := path.Split(inpath)
 	// Hard-code the sector type to 32GiBV1_1, because:
 	// - ffiwrapper.GeneratePieceCIDFromFile requires a RegisteredSealProof
@@ -36,7 +37,7 @@ func CalcCommP(ctx context.Context, inpath string, rename bool) (*CommPRet, erro
 	arbitraryProofType := abi.RegisteredSealProof_StackedDrg32GiBV1_1
 
 	st, err := os.Stat(inpath)
-	if err != nil  {
+	if err != nil {
 		return nil, err
 	}
 
@@ -45,7 +46,7 @@ func CalcCommP(ctx context.Context, inpath string, rename bool) (*CommPRet, erro
 	}
 	payloadSize := st.Size()
 
-	rdr, err := os.Open(inpath)
+	rdr, err := os.OpenFile(inpath, os.O_RDWR, 0644)
 	if err != nil {
 		return nil, err
 	}
@@ -55,7 +56,7 @@ func CalcCommP(ctx context.Context, inpath string, rename bool) (*CommPRet, erro
 	if err != nil {
 		return nil, err
 	}
-
+	carSize := stat.Size()
 	// check that the data is a car file; if it's not, retrieval won't work
 	_, err = car.ReadHeader(bufio.NewReader(rdr))
 	if err != nil {
@@ -66,7 +67,7 @@ func CalcCommP(ctx context.Context, inpath string, rename bool) (*CommPRet, erro
 		return nil, xerrors.Errorf("seek to start: %w", err)
 	}
 
-	pieceReader, pieceSize := padreader.New(rdr, uint64(stat.Size()))
+	pieceReader, pieceSize := padreader.New(rdr, uint64(carSize))
 	commP, err := ffiwrapper.GeneratePieceCIDFromFile(arbitraryProofType, pieceReader, pieceSize)
 	if err != nil {
 		return nil, xerrors.Errorf("computing commP failed: %w", err)
@@ -75,6 +76,16 @@ func CalcCommP(ctx context.Context, inpath string, rename bool) (*CommPRet, erro
 	if padreader.PaddedSize(uint64(payloadSize)) != pieceSize {
 		return nil, xerrors.Errorf("assert car(%s) file to piece fail payload size(%d) piece size (%d)", inpath, payloadSize, pieceSize)
 	}
+	if addPadding {
+		// make sure fd point to the end of file
+		// better to check within carv1.PadCar, for now is a workaround
+		if _, err := rdr.Seek(carSize, io.SeekStart); err != nil {
+			return nil, xerrors.Errorf("seek to start: %w", err)
+		}
+		if err := carv1.PadCar(rdr, carSize); err != nil {
+			return nil, xerrors.Errorf("failed to pad car file: %w", err)
+		}
+	}
 	if rename {
 		piecePath := path.Join(dir, commP.String())
 		err = os.Rename(inpath, piecePath)
@@ -83,9 +94,8 @@ func CalcCommP(ctx context.Context, inpath string, rename bool) (*CommPRet, erro
 		}
 	}
 	return &CommPRet{
-		Root: commP,
-		Size: pieceSize,
+		Root:        commP,
+		Size:        pieceSize,
 		PayloadSize: payloadSize,
 	}, nil
 }
-

commp_test.go

@@ -29,7 +29,7 @@ func TestCalcCommP(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	res, err := CalcCommP(context.TODO(), tempfile, false)
+	res, err := CalcCommP(context.TODO(), tempfile, false, false)
 	if err != nil {
 		t.Fatal(err)
 	}

go.mod

@@ -7,16 +7,17 @@ require (
 	github.com/filecoin-project/go-commp-utils v0.1.3
 	github.com/filecoin-project/go-padreader v0.0.1
 	github.com/filecoin-project/go-state-types v0.1.3
-	github.com/ipfs/go-blockservice v0.2.1
+	github.com/filedrive-team/filehelper v0.1.1
+	github.com/ipfs/go-blockservice v0.3.0
 	github.com/ipfs/go-cid v0.1.0
 	github.com/ipfs/go-datastore v0.5.1
-	github.com/ipfs/go-ipfs-blockstore v1.1.2
+	github.com/ipfs/go-ipfs-blockstore v1.2.0
 	github.com/ipfs/go-ipfs-chunker v0.0.5
-	github.com/ipfs/go-ipfs-exchange-offline v0.1.1
+	github.com/ipfs/go-ipfs-exchange-offline v0.2.0
 	github.com/ipfs/go-ipfs-files v0.0.9
-	github.com/ipfs/go-ipld-format v0.2.0
+	github.com/ipfs/go-ipld-format v0.4.0
 	github.com/ipfs/go-log/v2 v2.5.1
-	github.com/ipfs/go-merkledag v0.5.1
+	github.com/ipfs/go-merkledag v0.6.0
 	github.com/ipfs/go-unixfs v0.3.1
 	github.com/ipld/go-car v0.3.3
 	github.com/ipld/go-ipld-prime v0.16.0