From 3c028ac46af16479291f31c129597a34de3fbf4a Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 11 Oct 2016 12:21:47 -0700 Subject: [PATCH] add: Allow proper adding of multiple directories with a single add invocation License: MIT Signed-off-by: Jeromy --- commands/cli/parse.go | 2 +- commands/files/file.go | 4 ++ commands/files/linkfile.go | 7 +++- commands/files/multipartfile.go | 5 +++ commands/files/readerfile.go | 12 +++++- commands/files/serialfile.go | 23 +++++++--- commands/files/slicefile.go | 11 ++++- core/coreunix/add.go | 36 +++++++++------- importer/importer.go | 2 +- test/sharness/t0040-add-and-cat.sh | 22 ++++++---- test/sharness/t0046-add-multiple-dirs.sh | 53 ++++++++++++++++++++++++ 11 files changed, 143 insertions(+), 34 deletions(-) create mode 100755 test/sharness/t0046-add-multiple-dirs.sh diff --git a/commands/cli/parse.go b/commands/cli/parse.go index d69eee77618..2cb3313fea9 100644 --- a/commands/cli/parse.go +++ b/commands/cli/parse.go @@ -424,7 +424,7 @@ func appendFile(fpath string, argDef *cmds.Argument, recursive, hidden bool) (fi } } - return files.NewSerialFile(path.Base(fpath), fpath, hidden, stat) + return files.NewSerialFile(path.Base(fpath), fpath, hidden, true, stat) } // Inform the user if a file is waiting on input diff --git a/commands/files/file.go b/commands/files/file.go index c2185153c78..fd84bbc1343 100644 --- a/commands/files/file.go +++ b/commands/files/file.go @@ -30,6 +30,10 @@ type File interface { // (and therefor supports calling `Read` and `Close`) IsDirectory() bool + // IsRoot returns whether or not this file (or directory) is the 'root' of + // a given operation. + IsRoot() bool + // NextFile returns the next child file available (if the File is a // directory). It will return (nil, io.EOF) if no more files are // available. If the file is a regular file (not a directory), NextFile diff --git a/commands/files/linkfile.go b/commands/files/linkfile.go index 18466f4bd5f..c04908c19ed 100644 --- a/commands/files/linkfile.go +++ b/commands/files/linkfile.go @@ -11,11 +11,12 @@ type Symlink struct { path string Target string stat os.FileInfo + root bool reader io.Reader } -func NewLinkFile(name, path, target string, stat os.FileInfo) File { +func NewLinkFile(name, path, target string, stat os.FileInfo) *Symlink { return &Symlink{ name: name, path: path, @@ -48,3 +49,7 @@ func (f *Symlink) FullPath() string { func (f *Symlink) Read(b []byte) (int, error) { return f.reader.Read(b) } + +func (f *Symlink) IsRoot() bool { + return f.root +} diff --git a/commands/files/multipartfile.go b/commands/files/multipartfile.go index b71dd7fe600..7d90ecccb27 100644 --- a/commands/files/multipartfile.go +++ b/commands/files/multipartfile.go @@ -26,6 +26,7 @@ type MultipartFile struct { Part *multipart.Part Reader *multipart.Reader Mediatype string + Root bool } func NewFileFromPart(part *multipart.Part) (File, error) { @@ -105,3 +106,7 @@ func (f *MultipartFile) Close() error { } return f.Part.Close() } + +func (f *MultipartFile) IsRoot() bool { + return f.Root +} diff --git a/commands/files/readerfile.go b/commands/files/readerfile.go index 7458e82dd22..6fe23670495 100644 --- a/commands/files/readerfile.go +++ b/commands/files/readerfile.go @@ -13,10 +13,16 @@ type ReaderFile struct { fullpath string reader io.ReadCloser stat os.FileInfo + root bool } func NewReaderFile(filename, path string, reader io.ReadCloser, stat os.FileInfo) *ReaderFile { - return &ReaderFile{filename, path, reader, stat} + return &ReaderFile{ + filename: filename, + fullpath: path, + reader: reader, + stat: stat, + } } func (f *ReaderFile) IsDirectory() bool { @@ -53,3 +59,7 @@ func (f *ReaderFile) Size() (int64, error) { } return f.stat.Size(), nil } + +func (f *ReaderFile) IsRoot() bool { + return f.root +} diff --git a/commands/files/serialfile.go b/commands/files/serialfile.go index 22de5c1b484..0067b5843e1 100644 --- a/commands/files/serialfile.go +++ b/commands/files/serialfile.go @@ -20,16 +20,19 @@ type serialFile struct { stat os.FileInfo current *File handleHiddenFiles bool + root bool } -func NewSerialFile(name, path string, hidden bool, stat os.FileInfo) (File, error) { +func NewSerialFile(name, path string, hidden, root bool, stat os.FileInfo) (File, error) { switch mode := stat.Mode(); { case mode.IsRegular(): file, err := os.Open(path) if err != nil { return nil, err } - return NewReaderFile(name, path, file, stat), nil + rf := NewReaderFile(name, path, file, stat) + rf.root = root + return rf, nil case mode.IsDir(): // for directories, stat all of the contents first, so we know what files to // open when NextFile() is called @@ -37,13 +40,19 @@ func NewSerialFile(name, path string, hidden bool, stat os.FileInfo) (File, erro if err != nil { return nil, err } - return &serialFile{name, path, contents, stat, nil, hidden}, nil + if root { + name = path + } + return &serialFile{name, path, contents, stat, nil, hidden, root}, nil case mode&os.ModeSymlink != 0: target, err := os.Readlink(path) if err != nil { return nil, err } - return NewLinkFile(name, path, target, stat), nil + + lf := NewLinkFile(name, path, target, stat) + lf.root = root + return lf, nil default: return nil, fmt.Errorf("Unrecognized file type for %s: %s", name, mode.String()) } @@ -55,6 +64,10 @@ func (f *serialFile) IsDirectory() bool { return true } +func (f *serialFile) IsRoot() bool { + return f.root +} + func (f *serialFile) NextFile() (File, error) { // if a file was opened previously, close it err := f.Close() @@ -86,7 +99,7 @@ func (f *serialFile) NextFile() (File, error) { // recursively call the constructor on the next file // if it's a regular file, we will open it as a ReaderFile // if it's a directory, files in it will be opened serially - sf, err := NewSerialFile(fileName, filePath, f.handleHiddenFiles, stat) + sf, err := NewSerialFile(fileName, filePath, f.handleHiddenFiles, false, stat) if err != nil { return nil, err } diff --git a/commands/files/slicefile.go b/commands/files/slicefile.go index 8d18dcaa372..c3681504ad6 100644 --- a/commands/files/slicefile.go +++ b/commands/files/slicefile.go @@ -13,10 +13,15 @@ type SliceFile struct { path string files []File n int + root bool } func NewSliceFile(filename, path string, files []File) *SliceFile { - return &SliceFile{filename, path, files, 0} + return &SliceFile{ + filename: filename, + path: path, + files: files, + } } func (f *SliceFile) IsDirectory() bool { @@ -74,3 +79,7 @@ func (f *SliceFile) Size() (int64, error) { return size, nil } + +func (f *SliceFile) IsRoot() bool { + return f.root +} diff --git a/core/coreunix/add.go b/core/coreunix/add.go index 980a5ea66da..6bfb1872614 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -187,24 +187,30 @@ func (adder *Adder) Finalize() (*dag.Node, error) { return nil, err } - var name string - if !adder.Wrap { - name = rootNode.Links[0].Name - - dir, ok := adder.mr.GetValue().(*mfs.Directory) - if !ok { - return nil, fmt.Errorf("root is not a directory") - } - - root, err = dir.Child(name) + if adder.Wrap { + err = adder.outputDirs("", root) if err != nil { return nil, err } - } + } else { + for _, lnk := range rootNode.Links { + name := lnk.Name - err = adder.outputDirs(name, root) - if err != nil { - return nil, err + dir, ok := adder.mr.GetValue().(*mfs.Directory) + if !ok { + return nil, fmt.Errorf("root is not a directory") + } + + root, err = dir.Child(name) + if err != nil { + return nil, err + } + + err = adder.outputDirs(name, root) + if err != nil { + return nil, err + } + } } err = adder.mr.Close() @@ -272,7 +278,7 @@ func AddR(n *core.IpfsNode, root string) (key string, err error) { return "", err } - f, err := files.NewSerialFile(root, root, false, stat) + f, err := files.NewSerialFile(root, root, false, true, stat) if err != nil { return "", err } diff --git a/importer/importer.go b/importer/importer.go index 190500ba903..2d87949a07e 100644 --- a/importer/importer.go +++ b/importer/importer.go @@ -29,7 +29,7 @@ func BuildDagFromFile(fpath string, ds dag.DAGService) (*dag.Node, error) { return nil, fmt.Errorf("`%s` is a directory", fpath) } - f, err := files.NewSerialFile(fpath, fpath, false, stat) + f, err := files.NewSerialFile(fpath, fpath, false, true, stat) if err != nil { return nil, err } diff --git a/test/sharness/t0040-add-and-cat.sh b/test/sharness/t0040-add-and-cat.sh index adc930ec301..a1db0e59428 100755 --- a/test/sharness/t0040-add-and-cat.sh +++ b/test/sharness/t0040-add-and-cat.sh @@ -306,12 +306,14 @@ test_expect_success "'ipfs add -r' succeeds" ' ' test_expect_success "'ipfs add -r' output looks good" ' + MOUNTDIR="QmcgF3bEc3q9G9YfxCXWYaV5H7bDyDD1CrsmUDgSQPdY4n" && PLANETS="QmWSgS32xQEcXMeqd3YPJLrNBLSdsfYCep2U7CFkyrjXwY" && MARS="QmPrrHqJzto9m7SyiRzarwkqPcCSsKR2EB1AyqJfe8L8tN" && VENUS="QmU5kp3BH3B8tnWUU2Pikdb2maksBNkb92FHRr56hyghh4" && - echo "added $MARS planets/mars.txt" >expected && - echo "added $VENUS planets/venus.txt" >>expected && - echo "added $PLANETS planets" >>expected && + echo "added $MARS mountdir/planets/mars.txt" >expected && + echo "added $VENUS mountdir/planets/venus.txt" >>expected && + echo "added $PLANETS mountdir/planets" >>expected && + echo "added $MOUNTDIR mountdir" >>expected && test_cmp expected actual ' @@ -325,18 +327,20 @@ test_expect_success "'ipfs add -rn' succeeds" ' ' test_expect_success "'ipfs add -rn' output looks good" ' + MDMOONS="QmT8p74vZ2WUmN5bV6DThsEvVkiCESz5ZTT4xoFuF9r1My" && MOONS="QmVKvomp91nMih5j6hYBA8KjbiaYvEetU2Q7KvtZkLe9nQ" && EUROPA="Qmbjg7zWdqdMaK2BucPncJQDxiALExph5k3NkQv5RHpccu" && JUPITER="QmS5mZddhFPLWFX3w6FzAy9QxyYkaxvUpsWCtZ3r7jub9J" && SATURN="QmaMagZT4rTE7Nonw8KGSK4oe1bh533yhZrCo1HihSG8FK" && TITAN="QmZzppb9WHn552rmRqpPfgU5FEiHH6gDwi3MrB9cTdPwdb" && MERCURY="QmUJjVtnN8YEeYcS8VmUeWffTWhnMQAkk5DzZdKnPhqUdK" && - echo "added $EUROPA moons/jupiter/europa.txt" >expected && - echo "added $MERCURY moons/mercury.txt" >>expected && - echo "added $TITAN moons/saturn/titan.txt" >>expected && - echo "added $JUPITER moons/jupiter" >>expected && - echo "added $SATURN moons/saturn" >>expected && - echo "added $MOONS moons" >>expected && + echo "added $EUROPA mountdir/moons/jupiter/europa.txt" >expected && + echo "added $MERCURY mountdir/moons/mercury.txt" >>expected && + echo "added $TITAN mountdir/moons/saturn/titan.txt" >>expected && + echo "added $JUPITER mountdir/moons/jupiter" >>expected && + echo "added $SATURN mountdir/moons/saturn" >>expected && + echo "added $MOONS mountdir/moons" >>expected && + echo "added $MDMOONS mountdir" >>expected && test_cmp expected actual ' diff --git a/test/sharness/t0046-add-multiple-dirs.sh b/test/sharness/t0046-add-multiple-dirs.sh new file mode 100755 index 00000000000..31341263fad --- /dev/null +++ b/test/sharness/t0046-add-multiple-dirs.sh @@ -0,0 +1,53 @@ +#!/bin/sh +# +# Copyright (c) 2016 Jeromy Johnson +# MIT Licensed; see the LICENSE file in this repository. +# + +test_description="Test adding multiple directories" + + +. lib/test-lib.sh + +test_expect_success "make some setup directories" ' + mkdir -p a/foo/bar/fish && + mkdir -p a/foo/cats && + mkdir -p a/dogs/baz && + echo "stuff" > a/foo/bar/fish/food && + echo "blah blah" > a/foo/cats/meow && + echo "ipfs is cool" > a/dogs/baz/bazbaz && + + mkdir -p other/a/foo/cats && + mkdir -p other/a/ipfs && + echo "meoooowwww" > other/a/foo/cats/meow && + echo "hashing hashing" > other/a/ipfs/blah && + + mkdir -p something/example && + echo "example" > something/example/text +' + +test_add_many_r() { + test_expect_success "add multiple directories at the same time" ' + ipfs add -r a other/a something > add_out + ' + + test_expect_success "output looks correct" ' + grep "added QmU5Ekbnb9H8bLz6ECEnMDnkEZh5JoY9tXHGCBxnsLqsDg a/foo/cats/meow" add_out && + grep "added QmNwTsPq4BEBwuQ1aFUPZ6ux3P7GnupfQd1mBre19iSej5 other/a/foo/cats/meow" add_out && + grep "added QmTHApDiVaLa2yc8G6Jg6booheoFGffWuSKTqwQ4zjP6fQ a" add_out && + grep "added QmSsiWzY9yxtcRMsKvYivDKzBN8kftAgsVjpyGAxstj4Aw other" add_out && + grep "added QmWGDSqhr53etAVa35XBCmknWvMuuUU5VMS7zf3TYNnm89 something" add_out + ' +} + +test_init_ipfs + +test_add_many_r + +test_launch_ipfs_daemon + +test_add_many_r + +test_kill_ipfs_daemon + +test_done