Skip to content
This repository has been archived by the owner on May 24, 2024. It is now read-only.

Commit

Permalink
Change unixfs empty folder to dynamic folder (#6)
Browse files Browse the repository at this point in the history 
* Change unixfs empty folder to dynamic folder

* Adding tests
  • Loading branch information
@liuziba
liuziba authored Jul 1, 2022
1 parent 3c7b420 commit 2da2ec3
Show file tree
Hide file tree
Showing 7 changed files with 107,032 additions and 47 deletions.
6 changes: 3 additions & 3 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,14 @@ GLOBAL OPTIONS:
--help, -h show help (default: false)
```
The input file can be a text file that contains a list of file infomation. i.e.
The input file can be a text file that contains a list of file information SORTED by the path. i.e.
```json
[
{
"Path": "test/test.txt",
"Size": 4038,
"Start": 1000,
"End": 2000
"Start": 1000, # Inclusive
"End": 2000 # Exclusive
},
{
"Path": "test/test2.txt",
Expand Down
4 changes: 3 additions & 1 deletion generate-car.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ type Result struct {
DataCid string
PieceCid string
PieceSize uint64
CidMap map[string]util.CidMapValue
}

type Input []util.Finfo
Expand Down Expand Up @@ -117,7 +118,7 @@ func main() {
}
cp := new(commp.Calc)
writer := bufio.NewWriterSize(io.MultiWriter(carF, cp), BufSize)
ipld, cid, err := util.GenerateCar(ctx, input, parent, tmpDir, writer)
ipld, cid, cidMap, err := util.GenerateCar(ctx, input, parent, tmpDir, writer)
if err != nil {
return err
}
Expand Down Expand Up @@ -157,6 +158,7 @@ func main() {
DataCid: cid,
PieceCid: commCid.String(),
PieceSize: pieceSize,
CidMap: cidMap,
})
if err != nil {
return err
Expand Down
3 changes: 3 additions & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,13 @@ require (
)

require (
github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/ipfs/bbloom v0.0.4 // indirect
github.com/ipfs/go-bitfield v1.0.0 // indirect
github.com/ipfs/go-block-format v0.0.3 // indirect
github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
github.com/ipfs/go-ipfs-exchange-interface v0.1.0 // indirect
Expand Down Expand Up @@ -64,6 +66,7 @@ require (
go.uber.org/multierr v1.6.0 // indirect
go.uber.org/zap v1.19.1 // indirect
golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
google.golang.org/protobuf v1.27.1 // indirect
lukechampine.com/blake3 v1.1.6 // indirect
Expand Down
35 changes: 27 additions & 8 deletions spec/generate-car_spec.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
]
}
}
expectedCidMap = {""=>{"Cid"=>"bafybeiceqv4l23zs2766j3i2ros3zvatxmanelmnzk753ue525d6j4azgy", "IsDir"=>true}, "test.txt"=>{"Cid"=>"bafkreibpk2zxqxtsnbijltzvz5hnsjwgmr6i5dte5fiuri2fgcpf52gtpq", "IsDir"=>false}}
stdout = `./generate-car -i test/test.json -o test -p test -t tmpdir`
result = JSON.parse(stdout)
expectDataCid = JSON.parse(expectIpld)['Hash']
Expand All @@ -30,8 +31,8 @@
expectPieceSize = streamCommpResult.lines.find{|line|line.include?'Padded piece'}.strip.split(' ')[-2].to_i
expect(result['PieceCid']).to eq(expectCommp)
expect(result['PieceSize']).to eq(expectPieceSize)
puts JSON.pretty_generate(result['Ipld'])
expect(result['Ipld']).to eq(JSON.parse(expectIpld))
expect(result['CidMap']).to eq(expectedCidMap)
end

it 'should work for files with parent path' do
Expand All @@ -57,6 +58,7 @@
]
}
}
expectedCidMap = {""=>{"IsDir"=>true, "Cid"=>"bafybeigivse44aebx2ipou7fvvejjighko4yvtcbrhcuoc4wx6xr2gtdwe"}, "test"=>{"IsDir"=>true, "Cid"=>"bafybeiceqv4l23zs2766j3i2ros3zvatxmanelmnzk753ue525d6j4azgy"}, "test/test.txt"=>{"IsDir"=>false, "Cid"=>"bafkreibpk2zxqxtsnbijltzvz5hnsjwgmr6i5dte5fiuri2fgcpf52gtpq"}}
stdout = `./generate-car -i test/test.json -o test -p . -t tmpdir`
result = JSON.parse(stdout)
expectDataCid = JSON.parse(expectIpld)['Hash']
Expand All @@ -66,8 +68,8 @@
expectPieceSize = streamCommpResult.lines.find{|line|line.include?'Padded piece'}.strip.split(' ')[-2].to_i
expect(result['PieceCid']).to eq(expectCommp)
expect(result['PieceSize']).to eq(expectPieceSize)
puts JSON.pretty_generate(result['Ipld'])
expect(result['Ipld']).to eq(JSON.parse(expectIpld))
expect(result['CidMap']).to eq(expectedCidMap)
end

it 'should work with partial file' do
Expand All @@ -86,6 +88,7 @@
]
}
}
expectedCidMap = {""=>{"Cid"=>"bafybeiejb5tmssizbrxv2p5q5tx34g4d424zylpp6fucp5m7bwiyhqxnxa", "IsDir"=>true}, "test.txt"=>{"Cid"=>"bafkreihgspm7pi3bgf44lag72wmqkms27t2et7kbmkvt537p5h4drdgzse", "IsDir"=>false}}
stdout = `./generate-car -i test/test-partial.json -o test -p test -t tmpdir`
result = JSON.parse(stdout)
expectDataCid = JSON.parse(expectIpld)['Hash']
Expand All @@ -95,8 +98,8 @@
expectPieceSize = streamCommpResult.lines.find{|line|line.include?'Padded piece'}.strip.split(' ')[-2].to_i
expect(result['PieceCid']).to eq(expectCommp)
expect(result['PieceSize']).to eq(expectPieceSize)
puts JSON.pretty_generate(result['Ipld'])
expect(result['Ipld']).to eq(JSON.parse(expectIpld))
expect(result['CidMap']).to eq(expectedCidMap)
end

it 'should work with multiple files' do
Expand All @@ -121,6 +124,7 @@
]
}
}
expectedCidMap = {""=>{"IsDir"=>true, "Cid"=>"bafybeige75p3h2b72aufmsjgtwqppiuf67nvd25vzugpc3xs2ab5gmsfwq"}, "test.txt"=>{"IsDir"=>false, "Cid"=>"bafkreibpk2zxqxtsnbijltzvz5hnsjwgmr6i5dte5fiuri2fgcpf52gtpq"}, "test2.txt"=>{"IsDir"=>false, "Cid"=>"bafkreihrzwmwtzh7ax25ue4txzeuhrr77a3okuqagez243z4dcd2wlz4my"}}
stdout = `./generate-car -i test/test-multiple.json -o test -p test -t tmpdir`
result = JSON.parse(stdout)
expectDataCid = JSON.parse(expectIpld)['Hash']
Expand All @@ -130,8 +134,8 @@
expectPieceSize = streamCommpResult.lines.find{|line|line.include?'Padded piece'}.strip.split(' ')[-2].to_i
expect(result['PieceCid']).to eq(expectCommp)
expect(result['PieceSize']).to eq(expectPieceSize)
puts JSON.pretty_generate(result['Ipld'])
expect(result['Ipld']).to eq(JSON.parse(expectIpld))
expect(result['CidMap']).to eq(expectedCidMap)
end

it 'should work with file with wrong but larger size' do
Expand Down Expand Up @@ -159,7 +163,6 @@
expectPieceSize = streamCommpResult.lines.find{|line|line.include?'Padded piece'}.strip.split(' ')[-2].to_i
expect(result['PieceCid']).to eq(expectCommp)
expect(result['PieceSize']).to eq(expectPieceSize)
puts JSON.pretty_generate(result['Ipld'])
expect(result['Ipld']).to eq(JSON.parse(expectIpld))
end

Expand Down Expand Up @@ -189,7 +192,6 @@
expectPieceSize = streamCommpResult.lines.find{|line|line.include?'Padded piece'}.strip.split(' ')[-2].to_i
expect(result['PieceCid']).to eq(expectCommp)
expect(result['PieceSize']).to eq(expectPieceSize)
puts JSON.pretty_generate(result['Ipld'])
expect(result['Ipld']).to eq(JSON.parse(expectIpld))
end
it 'should work with symbolic link' do
Expand Down Expand Up @@ -217,7 +219,6 @@
expectPieceSize = streamCommpResult.lines.find{|line|line.include?'Padded piece'}.strip.split(' ')[-2].to_i
expect(result['PieceCid']).to eq(expectCommp)
expect(result['PieceSize']).to eq(expectPieceSize)
puts JSON.pretty_generate(result['Ipld'])
expect(result['Ipld']).to eq(JSON.parse(expectIpld))
end

Expand All @@ -233,6 +234,23 @@
expect($?.exitstatus).to eq(1)
end

it 'should work with large number of sub files' do
base = 'subfiles_test'
FileUtils.mkdir_p(File.join(base, 'subfolders'))
json = (10000..20000).map do |i|
path = File.join(base, "subfolders/#{i}.txt")
File.write(path, "Hello World #{i}")
{
"Path" => path,
"Size" => File.size(path)
}
end
File.write('subfiles_test/test.json', JSON.generate(json))
stdout = `./generate-car -i subfiles_test/test.json -o test -p subfiles_test`
result = JSON.parse(stdout)
expect(JSON.generate(result)).to eq(JSON.generate(JSON.parse(File.read('test/test-dynamic-folder.json'))))
end

it 'should handle complicated file structure' do
base = 'generated_test'
FileUtils.mkdir_p(base)
Expand Down Expand Up @@ -356,6 +374,7 @@
]
}
}
expectedCidMap = {""=>{"IsDir"=>true, "Cid"=>"bafybeib3roh4zlejbijx3ub2pkvko6szvo3yncqpxh53jlcwabsscvntgy"}, "1"=>{"IsDir"=>true, "Cid"=>"bafybeihul4g4is36a2lvqwd4osksjzcyqr6q3kj6kmlycwgsvrqwvkpuzq"}, "1/1"=>{"IsDir"=>true, "Cid"=>"bafybeidyv33hivd5ll27m6y7caaskhurr6twoobkl57xrugwpwmr47rcea"}, "1/1/1.txt"=>{"IsDir"=>false, "Cid"=>"bafkreibpk2zxqxtsnbijltzvz5hnsjwgmr6i5dte5fiuri2fgcpf52gtpq"}, "1/1/2.txt"=>{"IsDir"=>false, "Cid"=>"bafkreibpk2zxqxtsnbijltzvz5hnsjwgmr6i5dte5fiuri2fgcpf52gtpq"}, "1/2.txt"=>{"IsDir"=>false, "Cid"=>"bafkreibpk2zxqxtsnbijltzvz5hnsjwgmr6i5dte5fiuri2fgcpf52gtpq"}, "1/3.txt"=>{"IsDir"=>false, "Cid"=>"bafkreibpk2zxqxtsnbijltzvz5hnsjwgmr6i5dte5fiuri2fgcpf52gtpq"}, "2"=>{"IsDir"=>true, "Cid"=>"bafybeigjw26ddnnmqunl4ezbypr7gr55wo7uznlcfh6yp6dt3jmcvnagbi"}, "2/1"=>{"IsDir"=>true, "Cid"=>"bafybeibue5osgywpuwy5waov7cjrdlxrxps3m2znjznm7l73x362attysq"}, "2/1/1.txt"=>{"IsDir"=>false, "Cid"=>"bafkreibpk2zxqxtsnbijltzvz5hnsjwgmr6i5dte5fiuri2fgcpf52gtpq"}, "2/1/2.txt"=>{"IsDir"=>false, "Cid"=>"bafkreibpk2zxqxtsnbijltzvz5hnsjwgmr6i5dte5fiuri2fgcpf52gtpq"}, "2/1/3"=>{"IsDir"=>true, "Cid"=>"bafybeiavjj7lncrdsu2d5rndsvsqfl6lg4p7hzyisb32il4qoybdtmirwi"}, "2/1/3/1"=>{"IsDir"=>true, "Cid"=>"bafybeib6gazg7coviekkoakcefjl7wachgqztr3utmw7uohg635ixthhmm"}, "2/1/3/1/1.txt"=>{"IsDir"=>false, "Cid"=>"bafkreibpk2zxqxtsnbijltzvz5hnsjwgmr6i5dte5fiuri2fgcpf52gtpq"}, "3.txt"=>{"IsDir"=>false, "Cid"=>"bafkreibpk2zxqxtsnbijltzvz5hnsjwgmr6i5dte5fiuri2fgcpf52gtpq"}}
stdout = `./generate-car -i generated_test/test.json -o test -p generated_test -t tmpdir`
result = JSON.parse(stdout)
expectDataCid = JSON.parse(expectIpld)['Hash']
Expand All @@ -365,7 +384,7 @@
expectPieceSize = streamCommpResult.lines.find{|line|line.include?'Padded piece'}.strip.split(' ')[-2].to_i
expect(result['PieceCid']).to eq(expectCommp)
expect(result['PieceSize']).to eq(expectPieceSize)
puts JSON.pretty_generate(result['Ipld'])
expect(result['Ipld']).to eq(JSON.parse(expectIpld))
expect(result['CidMap']).to eq(expectedCidMap)
end
end
Loading

0 comments on commit 2da2ec3

Please sign in to comment.