Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ls: Report real file size #5906

Merged
merged 4 commits into from
Jan 17, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 34 additions & 12 deletions core/commands/ls.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ type LsOutput struct {
const (
lsHeadersOptionNameTime = "headers"
lsResolveTypeOptionName = "resolve-type"
lsSizeOptionName = "size"
lsStreamOptionName = "stream"
)

Expand All @@ -66,6 +67,7 @@ The JSON output contains type information.
Options: []cmdkit.Option{
cmdkit.BoolOption(lsHeadersOptionNameTime, "v", "Print table headers (Hash, Size, Name)."),
cmdkit.BoolOption(lsResolveTypeOptionName, "Resolve linked objects to find out their types.").WithDefault(true),
cmdkit.BoolOption(lsSizeOptionName, "Resolve linked objects to find out their file size.").WithDefault(true),
cmdkit.BoolOption(lsStreamOptionName, "s", "Enable exprimental streaming of directory entries as they are traversed."),
},
Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
Expand All @@ -79,9 +81,10 @@ The JSON output contains type information.
return err
}

resolve, _ := req.Options[lsResolveTypeOptionName].(bool)
resolveType, _ := req.Options[lsResolveTypeOptionName].(bool)
resolveSize, _ := req.Options[lsSizeOptionName].(bool)
dserv := nd.DAG
if !resolve {
if !resolveType && !resolveSize {
offlineexch := offline.Exchange(nd.Blockstore)
bserv := blockservice.New(nd.Blockstore, offlineexch)
dserv = merkledag.NewDAGService(bserv)
Expand Down Expand Up @@ -131,7 +134,7 @@ The JSON output contains type information.
}
outputLinks := make([]LsLink, len(links))
for j, link := range links {
lsLink, err := makeLsLink(req, dserv, resolve, link)
lsLink, err := makeLsLink(req, dserv, resolveType, resolveSize, link)
if err != nil {
return err
}
Expand Down Expand Up @@ -165,7 +168,7 @@ The JSON output contains type information.
return linkResult.Err
}
link := linkResult.Link
lsLink, err := makeLsLink(req, dserv, resolve, link)
lsLink, err := makeLsLink(req, dserv, resolveType, resolveSize, link)
if err != nil {
return err
}
Expand Down Expand Up @@ -224,16 +227,18 @@ func makeDagNodeLinkResults(req *cmds.Request, dagnode ipld.Node) <-chan unixfs.
return linkResults
}

func makeLsLink(req *cmds.Request, dserv ipld.DAGService, resolve bool, link *ipld.Link) (*LsLink, error) {
func makeLsLink(req *cmds.Request, dserv ipld.DAGService, resolveType bool, resolveSize bool, link *ipld.Link) (*LsLink, error) {
t := unixfspb.Data_DataType(-1)
var size uint64

switch link.Cid.Type() {
case cid.Raw:
// No need to check with raw leaves
t = unixfs.TFile
size = link.Size
case cid.DagProtobuf:
linkNode, err := link.GetNode(req.Context, dserv)
if err == ipld.ErrNotFound && !resolve {
if err == ipld.ErrNotFound && !resolveType && !resolveSize {
// not an error
linkNode = nil
} else if err != nil {
Expand All @@ -245,20 +250,26 @@ func makeLsLink(req *cmds.Request, dserv ipld.DAGService, resolve bool, link *ip
if err != nil {
return nil, err
}
t = d.Type()
if resolveType {
t = d.Type()
}
if d.Type() == unixfs.TFile && resolveSize {
size = d.FileSize()
}
}
}
return &LsLink{
Name: link.Name,
Hash: link.Cid.String(),
Size: link.Size,
Size: size,
Type: t,
}, nil
}

func tabularOutput(req *cmds.Request, w io.Writer, out *LsOutput, lastObjectHash string, ignoreBreaks bool) string {
headers, _ := req.Options[lsHeadersOptionNameTime].(bool)
stream, _ := req.Options[lsStreamOptionName].(bool)
size, _ := req.Options[lsSizeOptionName].(bool)
// in streaming mode we can't automatically align the tabs
// so we take a best guess
var minTabWidth int
Expand All @@ -282,17 +293,28 @@ func tabularOutput(req *cmds.Request, w io.Writer, out *LsOutput, lastObjectHash
fmt.Fprintf(tw, "%s:\n", object.Hash)
}
if headers {
fmt.Fprintln(tw, "Hash\tSize\tName")
s := "Hash\tName"
if size {
s = "Hash\tSize\tName"
}
fmt.Fprintln(tw, s)
}
lastObjectHash = object.Hash
}

for _, link := range object.Links {
if link.Type == unixfs.TDirectory {
link.Name += "/"
s := "%[1]s\t%[3]s\n"

switch {
case link.Type == unixfs.TDirectory && size:
s = "%[1]s\t-\t%[3]s/\n"
case link.Type == unixfs.TDirectory && !size:
s = "%[1]s\t%[3]s/\n"
case size:
s = "%s\t%v\t%s\n"
}

fmt.Fprintf(tw, "%s\t%v\t%s\n", link.Hash, link.Size, link.Name)
fmt.Fprintf(tw, s, link.Hash, link.Size, link.Name)
}
}
tw.Flush()
Expand Down
147 changes: 106 additions & 41 deletions test/sharness/t0045-ls.sh
Original file line number Diff line number Diff line change
Expand Up @@ -45,18 +45,41 @@ EOF
test_expect_success "'ipfs ls <three dir hashes>' output looks good" '
cat <<-\EOF >expected_ls &&
QmfNy183bXiRVyrhyWtq3TwHn79yHEkiAGFr18P7YNzESj:
QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss 246 d1/
QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy 1143 d2/
QmeomffUNfmQy76CQGy9NdmqEnnHU9soCexBnGU3ezPHVH 13 f1
QmNtocSs7MoDkJMc1RkyisCSKvLadujPsfJfSdJ3e1eA1M 13 f2
QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss - d1/
QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy - d2/
QmeomffUNfmQy76CQGy9NdmqEnnHU9soCexBnGU3ezPHVH 5 f1
QmNtocSs7MoDkJMc1RkyisCSKvLadujPsfJfSdJ3e1eA1M 5 f2

QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy:
QmbQBUSRL9raZtNXfpTDeaxQapibJEG6qEY8WqAN22aUzd 1035 1024
QmaRGe7bVmVaLmxbrMiVNXqW4pRNNp3xq7hFtyRKA3mtJL 14 a
QmbQBUSRL9raZtNXfpTDeaxQapibJEG6qEY8WqAN22aUzd 1024 1024
QmaRGe7bVmVaLmxbrMiVNXqW4pRNNp3xq7hFtyRKA3mtJL 6 a

QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss:
QmQNd6ubRXaNG6Prov8o6vk3bn6eWsj9FxLGrAVDUAGkGe 139 128
QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN 14 a
QmQNd6ubRXaNG6Prov8o6vk3bn6eWsj9FxLGrAVDUAGkGe 128 128
QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN 6 a
EOF
test_cmp expected_ls actual_ls
'

test_expect_success "'ipfs ls --size=false <three dir hashes>' succeeds" '
ipfs ls --size=false QmfNy183bXiRVyrhyWtq3TwHn79yHEkiAGFr18P7YNzESj QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss >actual_ls
'

test_expect_success "'ipfs ls <three dir hashes>' output looks good" '
cat <<-\EOF >expected_ls &&
QmfNy183bXiRVyrhyWtq3TwHn79yHEkiAGFr18P7YNzESj:
QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss d1/
QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy d2/
QmeomffUNfmQy76CQGy9NdmqEnnHU9soCexBnGU3ezPHVH f1
QmNtocSs7MoDkJMc1RkyisCSKvLadujPsfJfSdJ3e1eA1M f2

QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy:
QmbQBUSRL9raZtNXfpTDeaxQapibJEG6qEY8WqAN22aUzd 1024
QmaRGe7bVmVaLmxbrMiVNXqW4pRNNp3xq7hFtyRKA3mtJL a

QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss:
QmQNd6ubRXaNG6Prov8o6vk3bn6eWsj9FxLGrAVDUAGkGe 128
QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN a
EOF
test_cmp expected_ls actual_ls
'
Expand All @@ -69,20 +92,20 @@ EOF
cat <<-\EOF >expected_ls_headers &&
QmfNy183bXiRVyrhyWtq3TwHn79yHEkiAGFr18P7YNzESj:
Hash Size Name
QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss 246 d1/
QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy 1143 d2/
QmeomffUNfmQy76CQGy9NdmqEnnHU9soCexBnGU3ezPHVH 13 f1
QmNtocSs7MoDkJMc1RkyisCSKvLadujPsfJfSdJ3e1eA1M 13 f2
QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss - d1/
QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy - d2/
QmeomffUNfmQy76CQGy9NdmqEnnHU9soCexBnGU3ezPHVH 5 f1
QmNtocSs7MoDkJMc1RkyisCSKvLadujPsfJfSdJ3e1eA1M 5 f2

QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy:
Hash Size Name
QmbQBUSRL9raZtNXfpTDeaxQapibJEG6qEY8WqAN22aUzd 1035 1024
QmaRGe7bVmVaLmxbrMiVNXqW4pRNNp3xq7hFtyRKA3mtJL 14 a
QmbQBUSRL9raZtNXfpTDeaxQapibJEG6qEY8WqAN22aUzd 1024 1024
QmaRGe7bVmVaLmxbrMiVNXqW4pRNNp3xq7hFtyRKA3mtJL 6 a

QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss:
Hash Size Name
QmQNd6ubRXaNG6Prov8o6vk3bn6eWsj9FxLGrAVDUAGkGe 139 128
QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN 14 a
QmQNd6ubRXaNG6Prov8o6vk3bn6eWsj9FxLGrAVDUAGkGe 128 128
QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN 6 a
EOF
test_cmp expected_ls_headers actual_ls_headers
'
Expand Down Expand Up @@ -124,18 +147,41 @@ EOF
test_expect_success "'ipfs ls --stream <three dir hashes>' output looks good" '
cat <<-\EOF >expected_ls_stream &&
QmfNy183bXiRVyrhyWtq3TwHn79yHEkiAGFr18P7YNzESj:
QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss 246 d1/
QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy 1143 d2/
QmeomffUNfmQy76CQGy9NdmqEnnHU9soCexBnGU3ezPHVH 13 f1
QmNtocSs7MoDkJMc1RkyisCSKvLadujPsfJfSdJ3e1eA1M 13 f2
QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss - d1/
QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy - d2/
QmeomffUNfmQy76CQGy9NdmqEnnHU9soCexBnGU3ezPHVH 5 f1
QmNtocSs7MoDkJMc1RkyisCSKvLadujPsfJfSdJ3e1eA1M 5 f2

QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy:
QmbQBUSRL9raZtNXfpTDeaxQapibJEG6qEY8WqAN22aUzd 1024 1024
QmaRGe7bVmVaLmxbrMiVNXqW4pRNNp3xq7hFtyRKA3mtJL 6 a

QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss:
QmQNd6ubRXaNG6Prov8o6vk3bn6eWsj9FxLGrAVDUAGkGe 128 128
QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN 6 a
EOF
test_cmp expected_ls_stream actual_ls_stream
'

test_expect_success "'ipfs ls --size=false --stream <three dir hashes>' succeeds" '
ipfs ls --size=false --stream QmfNy183bXiRVyrhyWtq3TwHn79yHEkiAGFr18P7YNzESj QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss >actual_ls_stream
'

test_expect_success "'ipfs ls --size=false --stream <three dir hashes>' output looks good" '
cat <<-\EOF >expected_ls_stream &&
QmfNy183bXiRVyrhyWtq3TwHn79yHEkiAGFr18P7YNzESj:
QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss d1/
QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy d2/
QmeomffUNfmQy76CQGy9NdmqEnnHU9soCexBnGU3ezPHVH f1
QmNtocSs7MoDkJMc1RkyisCSKvLadujPsfJfSdJ3e1eA1M f2

QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy:
QmbQBUSRL9raZtNXfpTDeaxQapibJEG6qEY8WqAN22aUzd 1035 1024
QmaRGe7bVmVaLmxbrMiVNXqW4pRNNp3xq7hFtyRKA3mtJL 14 a
QmbQBUSRL9raZtNXfpTDeaxQapibJEG6qEY8WqAN22aUzd 1024
QmaRGe7bVmVaLmxbrMiVNXqW4pRNNp3xq7hFtyRKA3mtJL a

QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss:
QmQNd6ubRXaNG6Prov8o6vk3bn6eWsj9FxLGrAVDUAGkGe 139 128
QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN 14 a
QmQNd6ubRXaNG6Prov8o6vk3bn6eWsj9FxLGrAVDUAGkGe 128
QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN a
EOF
test_cmp expected_ls_stream actual_ls_stream
'
Expand All @@ -148,20 +194,20 @@ EOF
cat <<-\EOF >expected_ls_stream_headers &&
QmfNy183bXiRVyrhyWtq3TwHn79yHEkiAGFr18P7YNzESj:
Hash Size Name
QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss 246 d1/
QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy 1143 d2/
QmeomffUNfmQy76CQGy9NdmqEnnHU9soCexBnGU3ezPHVH 13 f1
QmNtocSs7MoDkJMc1RkyisCSKvLadujPsfJfSdJ3e1eA1M 13 f2
QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss - d1/
QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy - d2/
QmeomffUNfmQy76CQGy9NdmqEnnHU9soCexBnGU3ezPHVH 5 f1
QmNtocSs7MoDkJMc1RkyisCSKvLadujPsfJfSdJ3e1eA1M 5 f2

QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy:
Hash Size Name
QmbQBUSRL9raZtNXfpTDeaxQapibJEG6qEY8WqAN22aUzd 1035 1024
QmaRGe7bVmVaLmxbrMiVNXqW4pRNNp3xq7hFtyRKA3mtJL 14 a
QmbQBUSRL9raZtNXfpTDeaxQapibJEG6qEY8WqAN22aUzd 1024 1024
QmaRGe7bVmVaLmxbrMiVNXqW4pRNNp3xq7hFtyRKA3mtJL 6 a

QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss:
Hash Size Name
QmQNd6ubRXaNG6Prov8o6vk3bn6eWsj9FxLGrAVDUAGkGe 139 128
QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN 14 a
QmQNd6ubRXaNG6Prov8o6vk3bn6eWsj9FxLGrAVDUAGkGe 128 128
QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN 6 a
EOF
test_cmp expected_ls_stream_headers actual_ls_stream_headers
'
Expand All @@ -172,18 +218,27 @@ test_ls_cmd_raw_leaves() {
mkdir -p somedir &&
echo bar > somedir/foo &&
ipfs add --raw-leaves -r somedir/ > /dev/null &&
ipfs ls QmThNTdtKaVoCVrYmM5EBS6U3S5vfKFue2TxbxxAxRcKKE > ls-actual
ipfs ls '$1' QmThNTdtKaVoCVrYmM5EBS6U3S5vfKFue2TxbxxAxRcKKE > ls-actual
echo "zb2rhf6GzX4ckKZtjy8yy8iyq1KttCrRyqDedD6xubhY3sw2F 4 foo" > ls-expect
test_cmp ls-actual ls-expect
'
}

test_ls_object() {
test_expect_success "ipfs add medium size file then 'ipfs ls --size=false' works as expected" '
random 500000 2 > somefile &&
HASH=$(ipfs add somefile -q) &&
echo "QmPrM8S5T7Q3M8DQvQMS7m41m3Aq4jBjzAzvky5fH3xfr4 " > ls-expect &&
echo "QmdaAntAzQqqVMo4B8V69nkQd5d918YjHXUe2oF6hr72ri " >> ls-expect &&
ipfs ls --size=false $HASH > ls-actual &&
test_cmp ls-actual ls-expect
'

test_expect_success "ipfs add medium size file then 'ipfs ls' works as expected" '
random 500000 2 > somefile &&
HASH=$(ipfs add somefile -q) &&
echo "QmPrM8S5T7Q3M8DQvQMS7m41m3Aq4jBjzAzvky5fH3xfr4 262158 " > ls-expect &&
echo "QmdaAntAzQqqVMo4B8V69nkQd5d918YjHXUe2oF6hr72ri 237870 " >> ls-expect &&
echo "QmPrM8S5T7Q3M8DQvQMS7m41m3Aq4jBjzAzvky5fH3xfr4 262144 " > ls-expect &&
echo "QmdaAntAzQqqVMo4B8V69nkQd5d918YjHXUe2oF6hr72ri 237856 " >> ls-expect &&
ipfs ls $HASH > ls-actual &&
test_cmp ls-actual ls-expect
'
Expand All @@ -193,13 +248,15 @@ test_ls_object() {
test_ls_cmd
test_ls_cmd_streaming
test_ls_cmd_raw_leaves
test_ls_cmd_raw_leaves --size
test_ls_object

# should work online
test_launch_ipfs_daemon
test_ls_cmd
test_ls_cmd_streaming
test_ls_cmd_raw_leaves
test_ls_cmd_raw_leaves --size
test_kill_ipfs_daemon
test_ls_object

Expand All @@ -225,24 +282,32 @@ test_expect_success "remove a file in dir" '
ipfs block rm $FILE
'

test_expect_success "'ipfs ls --resolve-type=false ' ok" '
ipfs ls --resolve-type=false $DIR > /dev/null
test_expect_success "'ipfs ls --resolve-type=false ' fails" '
test_must_fail ipfs ls --resolve-type=false $DIR > /dev/null
'

test_expect_success "'ipfs ls' fails" '
test_must_fail ipfs ls $DIR
'

test_expect_success "'ipfs ls --resolve-type=true --size=false' fails" '
test_must_fail ipfs ls --resolve-type=true --size=false $DIR
'

test_launch_ipfs_daemon --offline

test_expect_success "'ipfs ls --resolve-type=false' ok" '
ipfs ls --resolve-type=false $DIR > /dev/null
test_expect_success "'ipfs ls --resolve-type=false --size=false' ok" '
ipfs ls --resolve-type=false --size=false $DIR > /dev/null
'

test_expect_success "'ipfs ls' fails" '
test_must_fail ipfs ls $DIR
'

test_expect_success "'ipfs ls --resolve-type=false --size=true' fails" '
test_must_fail ipfs ls --resolve-type=false --size=true $DIR
'

test_kill_ipfs_daemon

test_launch_ipfs_daemon
Expand All @@ -252,8 +317,8 @@ test_launch_ipfs_daemon
# it does it should eventually fail as the content is random and
# should not exist on the network, but we don't want to wait for a
# timeout so we will kill the request after a few seconds
test_expect_success "'ipfs ls --resolve-type=false' ok and does not hang" '
go-timeout 2 ipfs ls --resolve-type=false $DIR
test_expect_success "'ipfs ls --resolve-type=false --size=false' ok and does not hang" '
go-timeout 2 ipfs ls --resolve-type=false --size=false $DIR
'

test_kill_ipfs_daemon
Expand Down
2 changes: 1 addition & 1 deletion test/sharness/t0051-object.sh
Original file line number Diff line number Diff line change
Expand Up @@ -292,7 +292,7 @@ test_object_cmd() {
'

test_expect_success "output looks good" '
echo "QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn 4 foo/" > patched_exp &&
echo "QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn - foo/" > patched_exp &&
test_cmp patched_exp patched_output
'

Expand Down
Loading