Skip to content

Commit

Permalink
Merge pull request #1552 from ipfs/fix-tar-reader
Browse files Browse the repository at this point in the history 
get: fix halt bug + improvements
  • Loading branch information
@jbenet
jbenet committed Aug 5, 2015
2 parents 6944c73 + f105ce4 commit d1366cd
Show file tree
Hide file tree
Showing 5 changed files with 343 additions and 337 deletions.
113 changes: 73 additions & 40 deletions core/commands/get.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,52 +98,85 @@ may also specify the level of compression by specifying '-l=<1-9>'.
return
}

if archive, _, _ := req.Option("archive").Bool(); archive || cmplvl != gzip.NoCompression {
if archive && !strings.HasSuffix(outPath, ".tar") {
outPath += ".tar"
}
if cmplvl != gzip.NoCompression {
outPath += ".gz"
}
fmt.Printf("Saving archive to %s\n", outPath)

file, err := os.Create(outPath)
if err != nil {
res.SetError(err, cmds.ErrNormal)
return
}
defer file.Close()

bar := pb.New(0).SetUnits(pb.U_BYTES)
bar.Output = os.Stderr
pbReader := bar.NewProxyReader(outReader)
bar.Start()
defer bar.Finish()

if _, err := io.Copy(file, pbReader); err != nil {
res.SetError(err, cmds.ErrNormal)
return
}
archive, _, _ := req.Option("archive").Bool()

gw := getWriter{
Out: os.Stdout,
Err: os.Stderr,
Archive: archive,
Compression: cmplvl,
}

if err := gw.Write(outReader, outPath); err != nil {
res.SetError(err, cmds.ErrNormal)
return
}
},
}

fmt.Printf("Saving file(s) to %s\n", outPath)
func progressBarForReader(out io.Writer, r io.Reader) (*pb.ProgressBar, *pb.Reader) {
// setup bar reader
// TODO: get total length of files
bar := pb.New(0).SetUnits(pb.U_BYTES)
bar.Output = out
barR := bar.NewProxyReader(r)
return bar, barR
}

// TODO: get total length of files
bar := pb.New(0).SetUnits(pb.U_BYTES)
bar.Output = os.Stderr
type getWriter struct {
Out io.Writer // for output to user
Err io.Writer // for progress bar output

// wrap the reader with the progress bar proxy reader
reader := bar.NewProxyReader(outReader)
Archive bool
Compression int
}

bar.Start()
defer bar.Finish()
extractor := &tar.Extractor{outPath}
if err := extractor.Extract(reader); err != nil {
res.SetError(err, cmds.ErrNormal)
func (gw *getWriter) Write(r io.Reader, fpath string) error {
if gw.Archive || gw.Compression != gzip.NoCompression {
return gw.writeArchive(r, fpath)
}
return gw.writeExtracted(r, fpath)
}

func (gw *getWriter) writeArchive(r io.Reader, fpath string) error {
// adjust file name if tar
if gw.Archive {
if !strings.HasSuffix(fpath, ".tar") && !strings.HasSuffix(fpath, ".tar.gz") {
fpath += ".tar"
}
},
}

// adjust file name if gz
if gw.Compression != gzip.NoCompression {
if !strings.HasSuffix(fpath, ".gz") {
fpath += ".gz"
}
}

// create file
file, err := os.Create(fpath)
if err != nil {
return err
}
defer file.Close()

fmt.Fprintf(gw.Out, "Saving archive to %s\n", fpath)
bar, barR := progressBarForReader(gw.Err, r)
bar.Start()
defer bar.Finish()

_, err = io.Copy(file, barR)
return err
}

func (gw *getWriter) writeExtracted(r io.Reader, fpath string) error {
fmt.Fprintf(gw.Out, "Saving file(s) to %s\n", fpath)
bar, barR := progressBarForReader(gw.Err, r)
bar.Start()
defer bar.Finish()

extractor := &tar.Extractor{fpath}
return extractor.Extract(barR)
}

func getCompressOptions(req cmds.Request) (int, error) {
Expand All @@ -161,12 +194,12 @@ func getCompressOptions(req cmds.Request) (int, error) {
}

func get(ctx context.Context, node *core.IpfsNode, p path.Path, compression int) (io.Reader, error) {
dagnode, err := core.Resolve(ctx, node, p)
dn, err := core.Resolve(ctx, node, p)
if err != nil {
return nil, err
}

return utar.NewReader(ctx, p, node.DAG, dagnode, compression)
return utar.DagArchive(ctx, dn, p.String(), node.DAG, compression)
}

// getZip is equivalent to `ipfs getdag $hash | gzip`
Expand Down
134 changes: 67 additions & 67 deletions test/sharness/t0090-get.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,110 +14,110 @@ test_init_ipfs
test_get_cmd() {

test_expect_success "'ipfs get --help' succeeds" '
ipfs get --help >actual
ipfs get --help >actual
'

test_expect_success "'ipfs get --help' output looks good" '
egrep "ipfs get.*<ipfs-path>" actual >/dev/null ||
test_fsh cat actual
egrep "ipfs get.*<ipfs-path>" actual >/dev/null ||
test_fsh cat actual
'

test_expect_success "ipfs get succeeds" '
echo "Hello Worlds!" >data &&
HASH=`ipfs add -q data` &&
ipfs get "$HASH" >actual
echo "Hello Worlds!" >data &&
HASH=`ipfs add -q data` &&
ipfs get "$HASH" >actual
'

test_expect_success "ipfs get output looks good" '
printf "%s\n\n" "Saving file(s) to $HASH" >expected &&
test_cmp expected actual
printf "%s\n\n" "Saving file(s) to $HASH" >expected &&
test_cmp expected actual
'

test_expect_success "ipfs get file output looks good" '
test_cmp "$HASH" data
test_cmp "$HASH" data
'
test_expect_success "ipfs get errors when trying to overwrite a file" '
test_must_fail ipfs get "$HASH" >actual &&
rm "$HASH"

test_expect_success "ipfs get DOES NOT error when trying to overwrite a file" '
ipfs get "$HASH" >actual &&
rm "$HASH"
'

test_expect_success "ipfs get -a succeeds" '
ipfs get "$HASH" -a >actual
ipfs get "$HASH" -a >actual
'

test_expect_success "ipfs get -a output looks good" '
printf "%s\n\n" "Saving archive to $HASH.tar" >expected &&
test_cmp expected actual
printf "%s\n\n" "Saving archive to $HASH.tar" >expected &&
test_cmp expected actual
'

# TODO: determine why this fails
test_expect_failure "ipfs get -a archive output is valid" '
tar -xf "$HASH".tar &&
test_cmp "$HASH" data &&
rm "$HASH".tar &&
rm "$HASH"
tar -xf "$HASH".tar &&
test_cmp "$HASH" data &&
rm "$HASH".tar &&
rm "$HASH"
'

test_expect_success "ipfs get -a -C succeeds" '
ipfs get "$HASH" -a -C >actual
ipfs get "$HASH" -a -C >actual
'

test_expect_success "ipfs get -a -C output looks good" '
printf "%s\n\n" "Saving archive to $HASH.tar.gz" >expected &&
test_cmp expected actual
printf "%s\n\n" "Saving archive to $HASH.tar.gz" >expected &&
test_cmp expected actual
'

# TODO(mappum)
test_expect_failure "gzipped tar archive output is valid" '
tar -zxf "$HASH".tar.gz &&
test_cmp "$HASH" data &&
rm "$HASH".tar.gz &&
rm "$HASH"
tar -zxf "$HASH".tar.gz &&
test_cmp "$HASH" data &&
rm "$HASH".tar.gz &&
rm "$HASH"
'

test_expect_success "ipfs get succeeds (directory)" '
mkdir -p dir &&
touch dir/a &&
mkdir -p dir/b &&
echo "Hello, Worlds!" >dir/b/c &&
HASH2=`ipfs add -r -q dir | tail -n 1` &&
ipfs get "$HASH2" >actual
'
mkdir -p dir &&
touch dir/a &&
mkdir -p dir/b &&
echo "Hello, Worlds!" >dir/b/c &&
HASH2=`ipfs add -r -q dir | tail -n 1` &&
ipfs get "$HASH2" >actual
'

test_expect_success "ipfs get output looks good (directory)" '
printf "%s\n\n" "Saving file(s) to $HASH2" >expected &&
test_cmp expected actual
printf "%s\n\n" "Saving file(s) to $HASH2" >expected &&
test_cmp expected actual
'

test_expect_success "ipfs get output is valid (directory)" '
test_cmp dir/a "$HASH2"/a &&
test_cmp dir/b/c "$HASH2"/b/c &&
rm -r "$HASH2"
test_cmp dir/a "$HASH2"/a &&
test_cmp dir/b/c "$HASH2"/b/c &&
rm -r "$HASH2"
'

test_expect_success "ipfs get -a -C succeeds (directory)" '
ipfs get "$HASH2" -a -C >actual
ipfs get "$HASH2" -a -C >actual
'

test_expect_success "ipfs get -a -C output looks good (directory)" '
printf "%s\n\n" "Saving archive to $HASH2.tar.gz" >expected &&
test_cmp expected actual
printf "%s\n\n" "Saving archive to $HASH2.tar.gz" >expected &&
test_cmp expected actual
'

# TODO(mappum)
test_expect_failure "gzipped tar archive output is valid (directory)" '
tar -zxf "$HASH2".tar.gz &&
test_cmp dir/a "$HASH2"/a &&
test_cmp dir/b/c "$HASH2"/b/c &&
rm -r "$HASH2"
tar -zxf "$HASH2".tar.gz &&
test_cmp dir/a "$HASH2"/a &&
test_cmp dir/b/c "$HASH2"/b/c &&
rm -r "$HASH2"
'

test_expect_success "ipfs get ../.. should fail" '
echo "Error: invalid ipfs ref path" >expected &&
test_must_fail ipfs get ../.. 2>actual &&
test_cmp expected actual
'
test_expect_success "ipfs get ../.. should fail" '
echo "Error: invalid ipfs ref path" >expected &&
test_must_fail ipfs get ../.. 2>actual &&
test_cmp expected actual
'
}

# should work offline
Expand Down
Loading

0 comments on commit d1366cd

Please sign in to comment.