diff --git a/README.md b/README.md index 11ec6f2..dc7a407 100644 --- a/README.md +++ b/README.md @@ -76,17 +76,17 @@ GLOBAL OPTIONS: --input value, -i value This is a ndjson file containing the list of files to be included in the car archive. If not specified, use stdin instead. (default: "-") --piece-size value, -s value Target piece size, default to minimum possible value (default: 0) --out-dir value, -o value Output directory to save the car file (default: ".") - --parent value, -p value Parent path of the dataset --help, -h show help ``` The input file needs to be a ndjson file that contains a list of file information. The list should be sorted. - -If a file is split into multiple parts, they can be stitched together by using the same Path and proper Start and End values. - -If the Start and End value of those parts are not aligned, the file stitched may be corrupted. ```ndjson +# Path needs to be relative {"Path":"test/test.txt","Size":100,"Start":0,"End":100,"Cid":"bafkqaaa"} {"Path":"test/test2.txt","Size":500,"Start":0,"End":250,"Cid":"bafkqbbb"} {"Path":"test/test2.txt","Size":500,"Start":250,"End":500,"Cid":"bafkqccc"} ``` +If a file is split into multiple parts, they can be stitched together by using the same Path and proper Start and End values. + +If the Start and End value of those parts are not aligned, the file stitched may be corrupted. + diff --git a/cmd/generate-ipld-car/main.go b/cmd/generate-ipld-car/main.go index 176c4c1..4acfd28 100644 --- a/cmd/generate-ipld-car/main.go +++ b/cmd/generate-ipld-car/main.go @@ -48,17 +48,10 @@ func main() { Usage: "Output directory to save the car file", Value: ".", }, - &cli.StringFlag{ - Name: "parent", - Aliases: []string{"p"}, - Usage: "Parent path of the dataset", - Required: true, - }, }, Action: func(c *cli.Context) error { inputFile := c.String("input") pieceSizeInput := c.Uint64("piece-size") outDir := c.String("out-dir") - parent := c.String("parent") var in *os.File if inputFile == "-" { in = os.Stdin @@ -81,7 +74,7 @@ func main() { cp := new(commp.Calc) writer := bufio.NewWriterSize(io.MultiWriter(carF, cp), BufSize) - cid, err := util.GenerateIpldCar(context.TODO(), in, parent, writer) + cid, err := util.GenerateIpldCar(context.TODO(), in, writer) if err != nil { return errors.Wrap(err, "failed to generate car file") } diff --git a/cmd/generate-ipld-car/util/generate-ipld.go b/cmd/generate-ipld-car/util/generate-ipld.go index 03530fd..e636c10 100644 --- a/cmd/generate-ipld-car/util/generate-ipld.go +++ b/cmd/generate-ipld-car/util/generate-ipld.go @@ -16,7 +16,6 @@ import ( "github.com/ipld/go-car" "github.com/pkg/errors" "io" - "path/filepath" "strings" ) @@ -119,13 +118,8 @@ func getNode(ctx context.Context, entry *FsEntry, dagServ ipld.DAGService, fakeN return nil, nil, errors.New("invalid entry type") } -func GenerateIpldCar(ctx context.Context, input io.Reader, parent string, writer io.Writer) (cid.Cid, error) { +func GenerateIpldCar(ctx context.Context, input io.Reader, writer io.Writer) (cid.Cid, error) { scanner := bufio.NewScanner(input) - parentPath, err := filepath.Abs(parent) - if err != nil { - return cid.Undef, errors.Wrap(err, "failed to get absolute path of parent") - } - blockStore := bstore.NewBlockstore(datastore.NewMapDatastore()) dagServ := merkledag.NewDAGService(blockservice.New(blockStore, nil)) rootDir := FsEntry{ @@ -141,13 +135,8 @@ func GenerateIpldCar(ctx context.Context, input io.Reader, parent string, writer return cid.Undef, errors.Wrap(err, "failed to unmarshal json") } - fPath, err := filepath.Abs(finfo.Path) - if err != nil { - return cid.Undef, errors.Wrap(err, "failed to get absolute path of file") - } - - relPath, err := filepath.Rel(parentPath, fPath) - relSegments := strings.Split(relPath, string(filepath.Separator)) + relPath := finfo.Path + relSegments := strings.Split(relPath, "/") pos := &rootDir for i, seg := range relSegments { last := i == len(relSegments)-1 diff --git a/spec/generate-car_spec.rb b/spec/generate-car_spec.rb index 896c3be..101b1d0 100644 --- a/spec/generate-car_spec.rb +++ b/spec/generate-car_spec.rb @@ -71,7 +71,7 @@ def importToIpfs(path) end end - out = `./generate-ipld-car -i test_output/test.ndjson -o test_output -p test_ipld` + out = `./generate-ipld-car -i test_output/test.ndjson -o test_output` puts out result = JSON.parse(out) dataCid = result['DataCid'] @@ -86,7 +86,7 @@ def importToIpfs(path) system("ipfs get #{dataCid} -o test_ipld_out") # Compare test_ipld and test_ipld_out folder to make sure they are the same - expect(`diff -r test_ipld test_ipld_out`).to eq '' + expect(`diff -r test_ipld test_ipld_out/test_ipld`).to eq '' end end