diff --git a/.travis.yml b/.travis.yml index fdaa9998..14596449 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,4 +19,4 @@ script: - go build -v ./... - go test -count=1 -cover -race -v ./... - go vet ./... - - FILES=$(gofmt -s -l . zipfs sftpfs mem); if [[ -n "${FILES}" ]]; then echo "You have go format errors; gofmt your changes"; exit 1; fi + - FILES=$(gofmt -s -l . zipfs sftpfs mem tarfs); if [[ -n "${FILES}" ]]; then echo "You have go format errors; gofmt your changes"; exit 1; fi diff --git a/README.md b/README.md index acd49306..c3e807ae 100644 --- a/README.md +++ b/README.md @@ -380,7 +380,6 @@ The following is a short list of possible backends we hope someone will implement: * SSH -* TAR * S3 # About the project diff --git a/tarfs/file.go b/tarfs/file.go new file mode 100644 index 00000000..e1d63edc --- /dev/null +++ b/tarfs/file.go @@ -0,0 +1,144 @@ +package tarfs + +import ( + "archive/tar" + "bytes" + "os" + "path/filepath" + "sort" + "syscall" + + "github.com/spf13/afero" +) + +type File struct { + h *tar.Header + data *bytes.Reader + closed bool + fs *Fs +} + +func (f *File) Close() error { + if f.closed { + return afero.ErrFileClosed + } + + f.closed = true + f.h = nil + f.data = nil + f.fs = nil + + return nil +} + +func (f *File) Read(p []byte) (n int, err error) { + if f.closed { + return 0, afero.ErrFileClosed + } + + if f.h.Typeflag == tar.TypeDir { + return 0, syscall.EISDIR + } + + return f.data.Read(p) +} + +func (f *File) ReadAt(p []byte, off int64) (n int, err error) { + if f.closed { + return 0, afero.ErrFileClosed + } + + if f.h.Typeflag == tar.TypeDir { + return 0, syscall.EISDIR + } + + return f.data.ReadAt(p, off) +} + +func (f *File) Seek(offset int64, whence int) (int64, error) { + if f.closed { + return 0, afero.ErrFileClosed + } + + if f.h.Typeflag == tar.TypeDir { + return 0, syscall.EISDIR + } + + return f.data.Seek(offset, whence) +} + +func (f *File) Write(p []byte) (n int, err error) { return 0, syscall.EROFS } + +func (f *File) WriteAt(p []byte, off int64) (n int, err error) { return 0, syscall.EROFS } + +func (f *File) Name() string { + return filepath.Join(splitpath(f.h.Name)) +} + +func (f *File) getDirectoryNames() ([]string, error) { + d, ok := f.fs.files[f.Name()] + if !ok { + return nil, &os.PathError{Op: "readdir", Path: f.Name(), Err: syscall.ENOENT} + } + + var names []string + for n := range d { + names = append(names, n) + } + sort.Strings(names) + + return names, nil +} + +func (f *File) Readdir(count int) ([]os.FileInfo, error) { + if f.closed { + return nil, afero.ErrFileClosed + } + + if !f.h.FileInfo().IsDir() { + return nil, syscall.ENOTDIR + } + + names, err := f.getDirectoryNames() + if err != nil { + return nil, err + } + + d := f.fs.files[f.Name()] + var fi []os.FileInfo + for _, n := range names { + if n == "" { + continue + } + + f := d[n] + fi = append(fi, f.h.FileInfo()) + if count > 0 && len(fi) >= count { + break + } + } + + return fi, nil +} + +func (f *File) Readdirnames(n int) ([]string, error) { + fi, err := f.Readdir(n) + if err != nil { + return nil, err + } + + var names []string + for _, f := range fi { + names = append(names, f.Name()) + } + + return names, nil +} + +func (f *File) Stat() (os.FileInfo, error) { return f.h.FileInfo(), nil } + +func (f *File) Sync() error { return nil } + +func (f *File) Truncate(size int64) error { return syscall.EROFS } + +func (f *File) WriteString(s string) (ret int, err error) { return 0, syscall.EROFS } diff --git a/tarfs/fs.go b/tarfs/fs.go new file mode 100644 index 00000000..ba3212bf --- /dev/null +++ b/tarfs/fs.go @@ -0,0 +1,134 @@ +// package tarfs implements a read-only in-memory representation of a tar archive +package tarfs + +import ( + "archive/tar" + "bytes" + "io" + "os" + "path/filepath" + "syscall" + "time" + + "github.com/spf13/afero" +) + +type Fs struct { + files map[string]map[string]*File +} + +func splitpath(name string) (dir, file string) { + name = filepath.ToSlash(name) + if len(name) == 0 || name[0] != '/' { + name = "/" + name + } + name = filepath.Clean(name) + dir, file = filepath.Split(name) + dir = filepath.Clean(dir) + return +} + +func New(t *tar.Reader) *Fs { + fs := &Fs{files: make(map[string]map[string]*File)} + for { + hdr, err := t.Next() + if err == io.EOF { + break + } + if err != nil { + return nil + } + + d, f := splitpath(hdr.Name) + if _, ok := fs.files[d]; !ok { + fs.files[d] = make(map[string]*File) + } + + var buf bytes.Buffer + size, err := buf.ReadFrom(t) + if err != nil { + panic("tarfs: reading from tar:" + err.Error()) + } + + if size != hdr.Size { + panic("tarfs: size mismatch") + } + + file := &File{ + h: hdr, + data: bytes.NewReader(buf.Bytes()), + fs: fs, + } + fs.files[d][f] = file + + } + + // Add a pseudoroot + fs.files[afero.FilePathSeparator][""] = &File{ + h: &tar.Header{ + Name: afero.FilePathSeparator, + Typeflag: tar.TypeDir, + Size: 0, + }, + data: bytes.NewReader(nil), + fs: fs, + } + + return fs +} + +func (fs *Fs) Open(name string) (afero.File, error) { + d, f := splitpath(name) + if _, ok := fs.files[d]; !ok { + return nil, &os.PathError{Op: "open", Path: name, Err: syscall.ENOENT} + } + + file, ok := fs.files[d][f] + if !ok { + return nil, &os.PathError{Op: "open", Path: name, Err: syscall.ENOENT} + } + + nf := *file + + return &nf, nil +} + +func (fs *Fs) Name() string { return "tarfs" } + +func (fs *Fs) Create(name string) (afero.File, error) { return nil, syscall.EROFS } + +func (fs *Fs) Mkdir(name string, perm os.FileMode) error { return syscall.EROFS } + +func (fs *Fs) MkdirAll(path string, perm os.FileMode) error { return syscall.EROFS } + +func (fs *Fs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) { + if flag != os.O_RDONLY { + return nil, &os.PathError{Op: "open", Path: name, Err: syscall.EPERM} + } + + return fs.Open(name) +} + +func (fs *Fs) Remove(name string) error { return syscall.EROFS } + +func (fs *Fs) RemoveAll(path string) error { return syscall.EROFS } + +func (fs *Fs) Rename(oldname string, newname string) error { return syscall.EROFS } + +func (fs *Fs) Stat(name string) (os.FileInfo, error) { + d, f := splitpath(name) + if _, ok := fs.files[d]; !ok { + return nil, &os.PathError{Op: "stat", Path: name, Err: syscall.ENOENT} + } + + file, ok := fs.files[d][f] + if !ok { + return nil, &os.PathError{Op: "stat", Path: name, Err: syscall.ENOENT} + } + + return file.h.FileInfo(), nil +} + +func (fs *Fs) Chmod(name string, mode os.FileMode) error { return syscall.EROFS } + +func (fs *Fs) Chtimes(name string, atime time.Time, mtime time.Time) error { return syscall.EROFS } diff --git a/tarfs/tarfs_test.go b/tarfs/tarfs_test.go new file mode 100644 index 00000000..8d5eaab7 --- /dev/null +++ b/tarfs/tarfs_test.go @@ -0,0 +1,402 @@ +// Most of the tests are stolen from the zipfs implementation +package tarfs + +import ( + "archive/tar" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "reflect" + "syscall" + "testing" + + "github.com/spf13/afero" +) + +var files = []struct { + name string + exists bool + isdir bool + size int64 + content string + contentAt4k string +}{ + {"/", true, true, 0, "", ""}, + {"/sub", true, true, 0, "", ""}, + {"/sub/testDir2", true, true, 0, "", ""}, + {"/sub/testDir2/testFile", true, false, 8192, "cccccccc", "ccccdddd"}, + {"/testFile", true, false, 8192, "aaaaaaaa", "aaaabbbb"}, + {"/testDir1/testFile", true, false, 8192, "bbbbbbbb", "bbbbcccc"}, + + {"/nonExisting", false, false, 0, "", ""}, +} + +var dirs = []struct { + name string + children []string +}{ + {"/", []string{"sub", "testDir1", "testFile"}}, + {"/sub", []string{"testDir2"}}, + {"/sub/testDir2", []string{"testFile"}}, + {"/testDir1", []string{"testFile"}}, +} + +var afs *afero.Afero + +func TestMain(m *testing.M) { + tf, err := os.Open("testdata/t.tar") + if err != nil { + fmt.Print(err) + os.Exit(1) + } + + tfs := New(tar.NewReader(tf)) + afs = &afero.Afero{Fs: tfs} + os.Exit(m.Run()) +} + +func TestFsOpen(t *testing.T) { + for _, f := range files { + file, err := afs.Open(f.name) + if (err == nil) != f.exists { + t.Errorf("%v exists = %v, but got err = %v", f.name, f.exists, err) + } + + if !f.exists { + continue + } + if err != nil { + t.Fatalf("%v: %v", f.name, err) + } + + if file.Name() != filepath.FromSlash(f.name) { + t.Errorf("Name(), got %v, expected %v", file.Name(), filepath.FromSlash(f.name)) + } + + s, err := file.Stat() + if err != nil { + t.Fatalf("stat %v: got error '%v'", file.Name(), err) + } + + if isdir := s.IsDir(); isdir != f.isdir { + t.Errorf("%v directory, got: %v, expected: %v", file.Name(), isdir, f.isdir) + } + + if size := s.Size(); size != f.size { + t.Errorf("%v size, got: %v, expected: %v", file.Name(), size, f.size) + } + } +} + +func TestRead(t *testing.T) { + for _, f := range files { + if !f.exists { + continue + } + + file, err := afs.Open(f.name) + if err != nil { + t.Fatalf("opening %v: %v", f.name, err) + } + + buf := make([]byte, 8) + n, err := file.Read(buf) + if err != nil { + if f.isdir && (err != syscall.EISDIR) { + t.Errorf("%v got error %v, expected EISDIR", f.name, err) + } else if !f.isdir { + t.Errorf("%v: %v", f.name, err) + } + } else if n != 8 { + t.Errorf("%v: got %d read bytes, expected 8", f.name, n) + } else if string(buf) != f.content { + t.Errorf("%v: got <%s>, expected <%s>", f.name, f.content, string(buf)) + } + + } +} + +func TestReadAt(t *testing.T) { + for _, f := range files { + if !f.exists { + continue + } + + file, err := afs.Open(f.name) + if err != nil { + t.Fatalf("opening %v: %v", f.name, err) + } + + buf := make([]byte, 8) + n, err := file.ReadAt(buf, 4092) + if err != nil { + if f.isdir && (err != syscall.EISDIR) { + t.Errorf("%v got error %v, expected EISDIR", f.name, err) + } else if !f.isdir { + t.Errorf("%v: %v", f.name, err) + } + } else if n != 8 { + t.Errorf("%v: got %d read bytes, expected 8", f.name, n) + } else if string(buf) != f.contentAt4k { + t.Errorf("%v: got <%s>, expected <%s>", f.name, f.contentAt4k, string(buf)) + } + + } +} + +func TestSeek(t *testing.T) { + for _, f := range files { + if !f.exists { + continue + } + + file, err := afs.Open(f.name) + if err != nil { + t.Fatalf("opening %v: %v", f.name, err) + } + + var tests = []struct { + offin int64 + whence int + offout int64 + }{ + {0, io.SeekStart, 0}, + {10, io.SeekStart, 10}, + {1, io.SeekCurrent, 11}, + {10, io.SeekCurrent, 21}, + {0, io.SeekEnd, f.size}, + {-1, io.SeekEnd, f.size - 1}, + } + + for _, s := range tests { + n, err := file.Seek(s.offin, s.whence) + if err != nil { + if f.isdir && err == syscall.EISDIR { + continue + } + + t.Errorf("%v: %v", f.name, err) + } + + if n != s.offout { + t.Errorf("%v: (off: %v, whence: %v): got %v, expected %v", f.name, s.offin, s.whence, n, s.offout) + } + } + + } +} + +func TestName(t *testing.T) { + for _, f := range files { + if !f.exists { + continue + } + + file, err := afs.Open(f.name) + if err != nil { + t.Fatalf("opening %v: %v", f.name, err) + } + + n := file.Name() + if n != filepath.FromSlash(f.name) { + t.Errorf("got: %v, expected: %v", n, filepath.FromSlash(f.name)) + } + + } +} + +func TestClose(t *testing.T) { + for _, f := range files { + if !f.exists { + continue + } + + file, err := afs.Open(f.name) + if err != nil { + t.Fatalf("opening %v: %v", f.name, err) + } + + err = file.Close() + if err != nil { + t.Errorf("%v: %v", f.name, err) + } + + err = file.Close() + if err == nil { + t.Errorf("%v: closing twice should return an error", f.name) + } + + buf := make([]byte, 8) + n, err := file.Read(buf) + if n != 0 || err == nil { + t.Errorf("%v: could read from a closed file", f.name) + } + + n, err = file.ReadAt(buf, 256) + if n != 0 || err == nil { + t.Errorf("%v: could readAt from a closed file", f.name) + } + + off, err := file.Seek(0, io.SeekStart) + if off != 0 || err == nil { + t.Errorf("%v: could seek from a closed file", f.name) + } + } +} + +func TestOpenFile(t *testing.T) { + for _, f := range files { + file, err := afs.OpenFile(f.name, os.O_RDONLY, 0400) + if !f.exists { + if !errors.Is(err, syscall.ENOENT) { + t.Errorf("%v: got %v, expected%v", f.name, err, syscall.ENOENT) + } + + continue + } + + if err != nil { + t.Fatalf("%v: %v", f.name, err) + } + file.Close() + + file, err = afs.OpenFile(f.name, os.O_CREATE, 0600) + if !errors.Is(err, syscall.EPERM) { + t.Errorf("%v: open for write: got %v, expected %v", f.name, err, syscall.EPERM) + } + + } +} + +func TestFsStat(t *testing.T) { + for _, f := range files { + fi, err := afs.Stat(f.name) + if !f.exists { + if !errors.Is(err, syscall.ENOENT) { + t.Errorf("%v: got %v, expected%v", f.name, err, syscall.ENOENT) + } + + continue + } + + if err != nil { + t.Fatalf("stat %v: got error '%v'", f.name, err) + } + + if isdir := fi.IsDir(); isdir != f.isdir { + t.Errorf("%v directory, got: %v, expected: %v", f.name, isdir, f.isdir) + } + + if size := fi.Size(); size != f.size { + t.Errorf("%v size, got: %v, expected: %v", f.name, size, f.size) + } + } +} + +func TestReaddir(t *testing.T) { + for _, d := range dirs { + dir, err := afs.Open(d.name) + if err != nil { + t.Fatal(err) + } + + fi, err := dir.Readdir(0) + if err != nil { + t.Fatal(err) + } + var names []string + for _, f := range fi { + names = append(names, f.Name()) + } + + if !reflect.DeepEqual(names, d.children) { + t.Errorf("%v: children, got '%v', expected '%v'", d.name, names, d.children) + } + + fi, err = dir.Readdir(1) + if err != nil { + t.Fatal(err) + } + + names = []string{} + for _, f := range fi { + names = append(names, f.Name()) + } + + if !reflect.DeepEqual(names, d.children[0:1]) { + t.Errorf("%v: children, got '%v', expected '%v'", d.name, names, d.children[0:1]) + } + } + + dir, err := afs.Open("/testFile") + if err != nil { + t.Fatal(err) + } + + _, err = dir.Readdir(-1) + if err != syscall.ENOTDIR { + t.Fatal("Expected error") + } +} + +func TestReaddirnames(t *testing.T) { + for _, d := range dirs { + dir, err := afs.Open(d.name) + if err != nil { + t.Fatal(err) + } + + names, err := dir.Readdirnames(0) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(names, d.children) { + t.Errorf("%v: children, got '%v', expected '%v'", d.name, names, d.children) + } + + names, err = dir.Readdirnames(1) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(names, d.children[0:1]) { + t.Errorf("%v: children, got '%v', expected '%v'", d.name, names, d.children[0:1]) + } + } + + dir, err := afs.Open("/testFile") + if err != nil { + t.Fatal(err) + } + + _, err = dir.Readdir(-1) + if err != syscall.ENOTDIR { + t.Fatal("Expected error") + } +} + +func TestGlob(t *testing.T) { + for _, s := range []struct { + glob string + entries []string + }{ + {filepath.FromSlash("/*"), []string{filepath.FromSlash("/sub"), filepath.FromSlash("/testDir1"), filepath.FromSlash("/testFile")}}, + {filepath.FromSlash("*"), []string{filepath.FromSlash("sub"), filepath.FromSlash("testDir1"), filepath.FromSlash("testFile")}}, + {filepath.FromSlash("sub/*"), []string{filepath.FromSlash("sub/testDir2")}}, + {filepath.FromSlash("sub/testDir2/*"), []string{filepath.FromSlash("sub/testDir2/testFile")}}, + {filepath.FromSlash("testDir1/*"), []string{filepath.FromSlash("testDir1/testFile")}}, + } { + entries, err := afero.Glob(afs.Fs, s.glob) + if err != nil { + t.Error(err) + } + if reflect.DeepEqual(entries, s.entries) { + t.Logf("glob: %s: glob ok", s.glob) + } else { + t.Errorf("glob: %s: got %#v, expected %#v", s.glob, entries, s.entries) + } + } +} diff --git a/tarfs/testdata/t.tar b/tarfs/testdata/t.tar new file mode 100644 index 00000000..d5b9aa0f Binary files /dev/null and b/tarfs/testdata/t.tar differ