Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add tarfs implementation #265

Closed
wants to merge 51 commits into from
Closed
Show file tree
Hide file tree
Changes from 46 commits
Commits
Show all changes
51 commits
Select commit Hold shift + click to select a range
a62d059
Initial commit for tarfs
agimenez Sep 8, 2020
b5d563f
tarfs: reword "open" status field
agimenez Sep 8, 2020
65021f4
tarfs: use TestMain for FS setup
agimenez Sep 8, 2020
b9649ab
tarfs: test: early exit for nonexisting files
agimenez Sep 8, 2020
dca4644
tarfs: create test for filesystem Open
agimenez Sep 8, 2020
3878f91
tarfs: implement File.Stat
agimenez Sep 8, 2020
240527d
tarfs: implement Fs.Open
agimenez Sep 8, 2020
70d0b41
tarfs: return error on non-supported methods
agimenez Sep 8, 2020
1313d42
tarfs: implement File.data as bytes.Reader
agimenez Sep 9, 2020
c3757a7
tarfs: short format for simple methods
agimenez Sep 9, 2020
4fa2df3
tarfs: add missing closing brace in tests
agimenez Sep 9, 2020
fd69b07
tarfs: add test for File.ReadAt
agimenez Sep 9, 2020
c00fca0
tarfs: test File.ReadAt
agimenez Sep 9, 2020
cf078ed
tarfs: add tests for File.Read
agimenez Sep 10, 2020
a6fc4ef
tarfs: implement File.Read
agimenez Sep 10, 2020
d2a31b6
tarfs: add tests for File.Seek
agimenez Sep 10, 2020
cb072ce
tarfs: implement File.Seek
agimenez Sep 10, 2020
5efe215
tarfs: add tests for File.Name
agimenez Sep 10, 2020
1964cda
tarfs: implement File.Name
agimenez Sep 10, 2020
2460424
tarfs: add tests for File.Close
agimenez Sep 10, 2020
de4ff61
tarfs: implement File.Close
agimenez Sep 10, 2020
2272432
tarfs: add tests for OpenFile
agimenez Sep 10, 2020
fa95f2a
tarfs: fix test for Fs.OpenFile
agimenez Sep 11, 2020
1263aae
tarfs: remove code not needed after using filepath.Clean
agimenez Sep 11, 2020
46a440e
tarfs: Open: return a copy of the internal structure
agimenez Sep 11, 2020
61466af
tarfs: implement Fs.OpenFile
agimenez Sep 11, 2020
18b2552
tarfs: use Fatalf for unexpected error in TestFsOpen
agimenez Sep 11, 2020
668fe50
tarfs: add tests for Fs.Stat
agimenez Sep 11, 2020
ff11db1
tarfs: implement Fs.Stat
agimenez Sep 11, 2020
aeb4371
tarfs: remove TestNewFs
agimenez Sep 11, 2020
e551b88
tarfs: remove unused code
agimenez Sep 11, 2020
2244d64
tarfs: change internal implementation
agimenez Sep 11, 2020
649d57c
tarfs: use Fatal errors to avoid panics
agimenez Sep 11, 2020
cade310
tarfs: add pseudoroot
agimenez Sep 11, 2020
b6929bb
tarfs: add tests for File.Readdir
agimenez Sep 11, 2020
b78e068
tarfs: add pointer Fs in the File structure
agimenez Sep 11, 2020
f32342c
tarfs: fix error
agimenez Sep 11, 2020
013b128
tarfs: use just the names for TestReaddir, easier than using fill os.…
agimenez Sep 12, 2020
009c013
tarfs: create a copy of the original entry when opening a file
agimenez Sep 12, 2020
fbc802a
tarfs: implement File.Readdir
agimenez Sep 12, 2020
94f7af9
tarfs: add tests for File.Readdirnames
agimenez Sep 12, 2020
e36f5a9
tarfs: implement Readdirnames
agimenez Sep 12, 2020
c24a1b7
tarfs: add test for File.Name
agimenez Sep 12, 2020
ac803ca
tarfs: change tests to use the Afero interface instead
agimenez Sep 12, 2020
059ba00
tarfs: add tests for Glob from zipfs
agimenez Sep 12, 2020
36bccae
tarfs: update main repo references to tarfs
agimenez Sep 12, 2020
0522d19
tarfs: use OS-specific file separator for pseudoroot
agimenez Sep 13, 2020
f6a3394
test name err for windows
agimenez Sep 13, 2020
6fc148c
fixup! test name err for windows
agimenez Sep 13, 2020
6238a5e
fixup! fixup! test name err for windows
agimenez Sep 13, 2020
2a62693
fixup! fixup! fixup! test name err for windows
agimenez Sep 13, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,4 @@ script:
- go build -v ./...
- go test -count=1 -cover -race -v ./...
- go vet ./...
- FILES=$(gofmt -s -l . zipfs sftpfs mem); if [[ -n "${FILES}" ]]; then echo "You have go format errors; gofmt your changes"; exit 1; fi
- FILES=$(gofmt -s -l . zipfs sftpfs mem tarfs); if [[ -n "${FILES}" ]]; then echo "You have go format errors; gofmt your changes"; exit 1; fi
1 change: 0 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -380,7 +380,6 @@ The following is a short list of possible backends we hope someone will
implement:

* SSH
* TAR
* S3

# About the project
Expand Down
143 changes: 143 additions & 0 deletions tarfs/file.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
package tarfs

import (
"archive/tar"
"bytes"
"os"
"sort"
"syscall"

"github.com/spf13/afero"
)

type File struct {
h *tar.Header
data *bytes.Reader
closed bool
fs *Fs
}

func (f *File) Close() error {
if f.closed {
return afero.ErrFileClosed
}

f.closed = true
f.h = nil
f.data = nil
f.fs = nil

return nil
}

func (f *File) Read(p []byte) (n int, err error) {
if f.closed {
return 0, afero.ErrFileClosed
}

if f.h.Typeflag == tar.TypeDir {
return 0, syscall.EISDIR
}

return f.data.Read(p)
}

func (f *File) ReadAt(p []byte, off int64) (n int, err error) {
if f.closed {
return 0, afero.ErrFileClosed
}

if f.h.Typeflag == tar.TypeDir {
return 0, syscall.EISDIR
}

return f.data.ReadAt(p, off)
}

func (f *File) Seek(offset int64, whence int) (int64, error) {
if f.closed {
return 0, afero.ErrFileClosed
}

if f.h.Typeflag == tar.TypeDir {
return 0, syscall.EISDIR
}

return f.data.Seek(offset, whence)
}

func (f *File) Write(p []byte) (n int, err error) { return 0, syscall.EROFS }

func (f *File) WriteAt(p []byte, off int64) (n int, err error) { return 0, syscall.EROFS }

func (f *File) Name() string {
return f.h.Name
}

func (f *File) getDirectoryNames() ([]string, error) {
d, ok := f.fs.files[f.Name()]
if !ok {
return nil, &os.PathError{Op: "readdir", Path: f.Name(), Err: syscall.ENOENT}
}

var names []string
for n := range d {
names = append(names, n)
}
sort.Strings(names)

return names, nil
}

func (f *File) Readdir(count int) ([]os.FileInfo, error) {
if f.closed {
return nil, afero.ErrFileClosed
}

if !f.h.FileInfo().IsDir() {
return nil, syscall.ENOTDIR
}

names, err := f.getDirectoryNames()
if err != nil {
return nil, err
}

d := f.fs.files[f.Name()]
var fi []os.FileInfo
for _, n := range names {
if n == "" {
continue
}

f := d[n]
fi = append(fi, f.h.FileInfo())
if count > 0 && len(fi) >= count {
break
}
}

return fi, nil
}

func (f *File) Readdirnames(n int) ([]string, error) {
fi, err := f.Readdir(n)
if err != nil {
return nil, err
}

var names []string
for _, f := range fi {
names = append(names, f.Name())
}

return names, nil
}

func (f *File) Stat() (os.FileInfo, error) { return f.h.FileInfo(), nil }

func (f *File) Sync() error { return nil }

func (f *File) Truncate(size int64) error { return syscall.EROFS }

func (f *File) WriteString(s string) (ret int, err error) { return 0, syscall.EROFS }
136 changes: 136 additions & 0 deletions tarfs/fs.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
// package tarfs implements a read-only in-memory representation of a tar archive
package tarfs

import (
"archive/tar"
"bytes"
"io"
"os"
"path/filepath"
"syscall"
"time"

"github.com/spf13/afero"
)

type Fs struct {
files map[string]map[string]*File
}

func splitpath(name string) (dir, file string) {
name = filepath.ToSlash(name)
if len(name) == 0 || name[0] != '/' {
name = "/" + name
}
name = filepath.Clean(name)
dir, file = filepath.Split(name)
dir = filepath.Clean(dir)
return
}

func New(t *tar.Reader) *Fs {
fs := &Fs{files: make(map[string]map[string]*File)}
for {
hdr, err := t.Next()
if err == io.EOF {
break
}
if err != nil {
return nil
}

d, f := splitpath(hdr.Name)
if _, ok := fs.files[d]; !ok {
fs.files[d] = make(map[string]*File)
}

var buf bytes.Buffer
size, err := buf.ReadFrom(t)
if err != nil {
panic("tarfs: reading from tar:" + err.Error())
}

if size != hdr.Size {
panic("tarfs: size mismatch")
}

file := &File{
h: hdr,
data: bytes.NewReader(buf.Bytes()),
fs: fs,
}
file.h.Name = filepath.Join(d, f)

fs.files[d][f] = file

}

// Add a pseudoroot
fs.files["/"][""] = &File{
h: &tar.Header{
Name: "/",
Typeflag: tar.TypeDir,
Size: 0,
},
data: bytes.NewReader(nil),
fs: fs,
}

return fs
}

func (fs *Fs) Open(name string) (afero.File, error) {
d, f := splitpath(name)
if _, ok := fs.files[d]; !ok {
return nil, &os.PathError{Op: "open", Path: name, Err: syscall.ENOENT}
}

file, ok := fs.files[d][f]
if !ok {
return nil, &os.PathError{Op: "open", Path: name, Err: syscall.ENOENT}
}

nf := *file

return &nf, nil
}

func (fs *Fs) Name() string { return "tarfs" }

func (fs *Fs) Create(name string) (afero.File, error) { return nil, syscall.EROFS }

func (fs *Fs) Mkdir(name string, perm os.FileMode) error { return syscall.EROFS }

func (fs *Fs) MkdirAll(path string, perm os.FileMode) error { return syscall.EROFS }

func (fs *Fs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) {
if flag != os.O_RDONLY {
return nil, &os.PathError{Op: "open", Path: name, Err: syscall.EPERM}
}

return fs.Open(name)
}

func (fs *Fs) Remove(name string) error { return syscall.EROFS }

func (fs *Fs) RemoveAll(path string) error { return syscall.EROFS }

func (fs *Fs) Rename(oldname string, newname string) error { return syscall.EROFS }

func (fs *Fs) Stat(name string) (os.FileInfo, error) {
d, f := splitpath(name)
if _, ok := fs.files[d]; !ok {
return nil, &os.PathError{Op: "stat", Path: name, Err: syscall.ENOENT}
}

file, ok := fs.files[d][f]
if !ok {
return nil, &os.PathError{Op: "stat", Path: name, Err: syscall.ENOENT}
}

return file.h.FileInfo(), nil
}

func (fs *Fs) Chmod(name string, mode os.FileMode) error { return syscall.EROFS }

func (fs *Fs) Chtimes(name string, atime time.Time, mtime time.Time) error { return syscall.EROFS }
Loading