Skip to content
This repository has been archived by the owner on May 24, 2024. It is now read-only.

Assume generate-ipld-car uses relative path as input #14

Merged
merged 2 commits into from
Mar 10, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -76,17 +76,17 @@ GLOBAL OPTIONS:
--input value, -i value This is a ndjson file containing the list of files to be included in the car archive. If not specified, use stdin instead. (default: "-")
--piece-size value, -s value Target piece size, default to minimum possible value (default: 0)
--out-dir value, -o value Output directory to save the car file (default: ".")
--parent value, -p value Parent path of the dataset
--help, -h show help
```

The input file needs to be a ndjson file that contains a list of file information. The list should be sorted.

If a file is split into multiple parts, they can be stitched together by using the same Path and proper Start and End values.

If the Start and End value of those parts are not aligned, the file stitched may be corrupted.
```ndjson
# Path needs to be relative
{"Path":"test/test.txt","Size":100,"Start":0,"End":100,"Cid":"bafkqaaa"}
{"Path":"test/test2.txt","Size":500,"Start":0,"End":250,"Cid":"bafkqbbb"}
{"Path":"test/test2.txt","Size":500,"Start":250,"End":500,"Cid":"bafkqccc"}
```
If a file is split into multiple parts, they can be stitched together by using the same Path and proper Start and End values.

If the Start and End value of those parts are not aligned, the file stitched may be corrupted.

9 changes: 1 addition & 8 deletions cmd/generate-ipld-car/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,17 +48,10 @@ func main() {
Usage: "Output directory to save the car file",
Value: ".",
},
&cli.StringFlag{
Name: "parent",
Aliases: []string{"p"},
Usage: "Parent path of the dataset",
Required: true,
},
}, Action: func(c *cli.Context) error {
inputFile := c.String("input")
pieceSizeInput := c.Uint64("piece-size")
outDir := c.String("out-dir")
parent := c.String("parent")
var in *os.File
if inputFile == "-" {
in = os.Stdin
Expand All @@ -81,7 +74,7 @@ func main() {

cp := new(commp.Calc)
writer := bufio.NewWriterSize(io.MultiWriter(carF, cp), BufSize)
cid, err := util.GenerateIpldCar(context.TODO(), in, parent, writer)
cid, err := util.GenerateIpldCar(context.TODO(), in, writer)
if err != nil {
return errors.Wrap(err, "failed to generate car file")
}
Expand Down
17 changes: 3 additions & 14 deletions cmd/generate-ipld-car/util/generate-ipld.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ import (
"github.com/ipld/go-car"
"github.com/pkg/errors"
"io"
"path/filepath"
"strings"
)

Expand Down Expand Up @@ -119,13 +118,8 @@ func getNode(ctx context.Context, entry *FsEntry, dagServ ipld.DAGService, fakeN
return nil, nil, errors.New("invalid entry type")
}

func GenerateIpldCar(ctx context.Context, input io.Reader, parent string, writer io.Writer) (cid.Cid, error) {
func GenerateIpldCar(ctx context.Context, input io.Reader, writer io.Writer) (cid.Cid, error) {
scanner := bufio.NewScanner(input)
parentPath, err := filepath.Abs(parent)
if err != nil {
return cid.Undef, errors.Wrap(err, "failed to get absolute path of parent")
}

blockStore := bstore.NewBlockstore(datastore.NewMapDatastore())
dagServ := merkledag.NewDAGService(blockservice.New(blockStore, nil))
rootDir := FsEntry{
Expand All @@ -141,13 +135,8 @@ func GenerateIpldCar(ctx context.Context, input io.Reader, parent string, writer
return cid.Undef, errors.Wrap(err, "failed to unmarshal json")
}

fPath, err := filepath.Abs(finfo.Path)
if err != nil {
return cid.Undef, errors.Wrap(err, "failed to get absolute path of file")
}

relPath, err := filepath.Rel(parentPath, fPath)
relSegments := strings.Split(relPath, string(filepath.Separator))
relPath := finfo.Path
relSegments := strings.Split(relPath, "/")
pos := &rootDir
for i, seg := range relSegments {
last := i == len(relSegments)-1
Expand Down
4 changes: 2 additions & 2 deletions spec/generate-car_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ def importToIpfs(path)
end
end

out = `./generate-ipld-car -i test_output/test.ndjson -o test_output -p test_ipld`
out = `./generate-ipld-car -i test_output/test.ndjson -o test_output`
puts out
result = JSON.parse(out)
dataCid = result['DataCid']
Expand All @@ -86,7 +86,7 @@ def importToIpfs(path)
system("ipfs get #{dataCid} -o test_ipld_out")

# Compare test_ipld and test_ipld_out folder to make sure they are the same
expect(`diff -r test_ipld test_ipld_out`).to eq ''
expect(`diff -r test_ipld test_ipld_out/test_ipld`).to eq ''
end
end

Expand Down