feat: support UnixFS 1.5 file mode and modification times

Replaces PR #34 written by @kstuart

Adds support for storing and retrieving file mode and last modification time.

Support added to:
- [X] Files
- [X] LinkFiles
- [X] Webfiles
- [X] Directories
- [X] Tar Archives

When the TAR archive (headers) include a file mode or modification time, the extractor will restore that metadata when supported for the underlying filesystem.

The Golang runtime currently does not support changing the times on a symlink, for Linux and some BSDs a custom solution has been implemented, for Darwin this is not the case so when copying a symlink to the filesystem the last modification time is not updated. Since for concrete files and directories stored modes and modification times are faithfully restored to the filesystem this should not be a breaking issue, a similar solution to that implemented for Linux/BSDs is likely implementable by a developer with access to a Darwin platform.

Replaces PRs:
- ipfs/go-ipfs-files#31
- ipfs/tar-utils#11
- #34

Relates to ipfs/kubo#6920

files/file.go

@@ -4,6 +4,7 @@ import (
  "errors"
  "io"
  "os"
+ "time"
 )
 
 var (
@@ -17,6 +18,13 @@ var (
 type Node interface {
  io.Closer
 
+ // Mode returns the file's mode.
+ Mode() os.FileMode
+
+ // ModTime returns the file's last modification time. If the last
+ // modification time is unknown/unspecified ModTime returns zero.
+ ModTime() (mtime time.Time)
+
  // Size returns size of this file (if this file is a directory, total size of
  // all files stored in the tree should be returned). Some implementations may
  // choose not to implement this

files/file_test.go

@@ -3,8 +3,10 @@ package files
 import (
  "io"
  "mime/multipart"
+ "os"
  "strings"
  "testing"
+ "time"
 )
 
 func TestSliceFiles(t *testing.T) {
@@ -49,6 +51,21 @@ func TestReaderFiles(t *testing.T) {
  }
 }
 
+func TestReaderFileStat(t *testing.T) {
+ reader := strings.NewReader("beep boop")
+ mode := os.FileMode(0754)
+ mtime := time.Date(2020, 11, 2, 12, 27, 35, 55555, time.UTC)
+ stat := &mockFileInfo{name: "test", mode: mode, mtime: mtime}
+
+ rf := NewReaderStatFile(reader, stat)
+ if rf.Mode() != mode {
+ t.Fatalf("Expected file mode to be [%v] but got [%v]", mode, rf.Mode())
+ }
+ if rf.ModTime() != mtime {
+ t.Fatalf("Expected file modified time to be [%v] but got [%v]", mtime, rf.ModTime())
+ }
+}
+
 func TestMultipartFiles(t *testing.T) {
  data := `
 --Boundary!
@@ -141,3 +158,127 @@ implicit file2
  },
  })
 }
+
+func TestMultipartFilesWithMode(t *testing.T) {
+ data := `
+--Boundary!
+Content-Type: text/plain
+Content-Disposition: form-data; name="file-0?mode=0754&mtime=1604320500&mtime-nsecs=55555"; filename="%C2%A3%E1%BA%9E%C7%91%C7%93%C3%86+%C3%A6+%E2%99%AB%E2%99%AC"
+Some-Header: beep
+
+beep
+--Boundary!
+Content-Type: application/x-directory
+Content-Disposition: form-data; name="dir-0?mode=755&mtime=1604320500"; ans=42; filename="dir1"
+
+--Boundary!
+Content-Type: text/plain
+Content-Disposition: form-data; name="file"; filename="dir1/nested"
+
+some content
+--Boundary!
+Content-Type: text/plain
+Content-Disposition: form-data; name="file?mode=600"; filename="dir1/nested2"; ans=42
+
+some content
+--Boundary!
+Content-Type: application/symlink
+Content-Disposition: form-data; name="file-5"; filename="dir1/simlynk"
+
+anotherfile
+--Boundary!
+Content-Type: application/symlink
+Content-Disposition: form-data; name="file?mtime=1604320500"; filename="dir1/simlynk2"
+
+anotherfile
+--Boundary!
+Content-Type: text/plain
+Content-Disposition: form-data; name="dir?mode=0644"; filename="implicit1/implicit2/deep_implicit"
+
+implicit file1
+--Boundary!
+Content-Type: text/plain
+Content-Disposition: form-data; name="dir?mode=755&mtime=1604320500"; filename="implicit1/shallow_implicit"
+
+implicit file2
+--Boundary!--
+
+`
+
+ reader := strings.NewReader(data)
+ mpReader := multipart.NewReader(reader, "Boundary!")
+ dir, err := NewFileFromPartReader(mpReader, multipartFormdataType)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ CheckDir(t, dir, []Event{
+ {
+ kind: TFile,
+ name: "£ẞǑǓÆ æ ♫♬",
+ value: "beep",
+ mode: 0754,
+ mtime: time.Unix(1604320500, 55555),
+ },
+ {
+ kind: TDirStart,
+ name: "dir1",
+ mode: 0755,
+ mtime: time.Unix(1604320500, 0),
+ },
+ {
+ kind: TFile,
+ name: "nested",
+ value: "some content",
+ },
+ {
+ kind: TFile,
+ name: "nested2",
+ value: "some content",
+ mode: 0600,
+ },
+ {
+ kind: TSymlink,
+ name: "simlynk",
+ value: "anotherfile",
+ mode: 0777,
+ },
+ {
+ kind: TSymlink,
+ name: "simlynk2",
+ value: "anotherfile",
+ mode: 0777,
+ mtime: time.Unix(1604320500, 0),
+ },
+ {
+ kind: TDirEnd,
+ },
+ {
+ kind: TDirStart,
+ name: "implicit1",
+ },
+ {
+ kind: TDirStart,
+ name: "implicit2",
+ },
+ {
+ kind: TFile,
+ name: "deep_implicit",
+ value: "implicit file1",
+ mode: 0644,
+ },
+ {
+ kind: TDirEnd,
+ },
+ {
+ kind: TFile,
+ name: "shallow_implicit",
+ value: "implicit file2",
+ mode: 0755,
+ mtime: time.Unix(1604320500, 0),
+ },
+ {
+ kind: TDirEnd,
+ },
+ })
+}

files/filter_test.go

@@ -4,17 +4,33 @@ import (
  "os"
  "path/filepath"
  "testing"
+ "time"
 )
 
 type mockFileInfo struct {
  os.FileInfo
- name string
+ name string
+ mode os.FileMode
+ mtime time.Time
+ size int64
 }
 
 func (m *mockFileInfo) Name() string {
  return m.name
 }
 
+func (m *mockFileInfo) Mode() os.FileMode {
+ return m.mode
+}
+
+func (m *mockFileInfo) ModTime() time.Time {
+ return m.mtime
+}
+
+func (m *mockFileInfo) Size() int64 {
+ return m.size
+}
+
 func (m *mockFileInfo) Sys() interface{} {
  return nil
 }

files/helpers_test.go

@@ -2,7 +2,9 @@ package files
 
 import (
  "io"
+ "os"
  "testing"
+ "time"
 )
 
 type Kind int
@@ -18,6 +20,8 @@ type Event struct {
  kind Kind
  name string
  value string
+ mode os.FileMode
+ mtime time.Time
 }
 
 func CheckDir(t *testing.T, dir Directory, expected []Event) {
@@ -50,6 +54,14 @@ func CheckDir(t *testing.T, dir Directory, expected []Event) {
  t.Fatalf("[%d] expected filename to be %q", i, next.name)
  }
 
+ if next.mode != 0 && it.Node().Mode()&os.ModePerm != next.mode {
+ t.Fatalf("[%d] expected mode for '%s' to be %O, got %O", i, it.Name(), next.mode, it.Node().Mode())
+ }
+
+ if !next.mtime.IsZero() && !it.Node().ModTime().Equal(next.mtime) {
+ t.Fatalf("[%d] expected modification time for '%s' to be %q", i, it.Name(), next.mtime)
+ }
+
  switch next.kind {
  case TFile:
  mf, ok := it.Node().(File)

files/linkfile.go

@@ -3,21 +3,41 @@ package files
 import (
  "os"
  "strings"
+ "time"
 )
 
 type Symlink struct {
  Target string
 
- stat os.FileInfo
+ mtime time.Time
  reader strings.Reader
 }
 
 func NewLinkFile(target string, stat os.FileInfo) File {
- lf := &Symlink{Target: target, stat: stat}
+ mtime := time.Time{}
+ if stat != nil {
+ mtime = stat.ModTime()
+ }
+ return NewSymlinkFile(target, mtime)
+}
+
+func NewSymlinkFile(target string, mtime time.Time) File {
+ lf := &Symlink{
+ Target: target,
+ mtime: mtime,
+ }
  lf.reader.Reset(lf.Target)
  return lf
 }
 
+func (lf *Symlink) Mode() os.FileMode {
+ return os.ModeSymlink | os.ModePerm
+}
+
+func (lf *Symlink) ModTime() time.Time {
+ return lf.mtime
+}
+
 func (lf *Symlink) Close() error {
  return nil
 }

files/multifilereader.go

@@ -8,6 +8,8 @@ import (
  "net/textproto"
  "net/url"
  "path"
+ "strconv"
+ "strings"
  "sync"
 )
 
@@ -89,18 +91,12 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) {
 
  // handle starting a new file part
  if !mfr.closed {
-
  mfr.currentFile = entry.Node()
 
  // write the boundary and headers
  header := make(textproto.MIMEHeader)
- filename := url.QueryEscape(path.Join(path.Join(mfr.path...), entry.Name()))
- dispositionPrefix := "attachment"
- if mfr.form {
- dispositionPrefix = "form-data; name=\"file\""
- }
-
- header.Set("Content-Disposition", fmt.Sprintf("%s; filename=\"%s\"", dispositionPrefix, filename))
+ filename := path.Join(path.Join(mfr.path...), entry.Name())
+ mfr.addContentDisposition(header, filename)
 
  var contentType string
 
@@ -119,7 +115,7 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) {
  return 0, ErrNotSupported
  }
 
- header.Set("Content-Type", contentType)
+ header.Set(contentTypeHeader, contentType)
  if rf, ok := entry.Node().(FileInfo); ok {
  if mfr.rawAbsPath {
  // Legacy compatibility with old servers.
@@ -157,6 +153,36 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) {
  return written, nil
 }
 
+func (mfr *MultiFileReader) addContentDisposition(header textproto.MIMEHeader, filename string) {
+ sb := &strings.Builder{}
+ params := url.Values{}
+
+ if mode := mfr.currentFile.Mode(); mode != 0 {
+ params.Add("mode", "0"+strconv.FormatUint(uint64(mode), 8))
+ }
+ if mtime := mfr.currentFile.ModTime(); !mtime.IsZero() {
+ params.Add("mtime", strconv.FormatInt(mtime.Unix(), 10))
+ if n := mtime.Nanosecond(); n > 0 {
+ params.Add("mtime-nsecs", strconv.FormatInt(int64(n), 10))
+ }
+ }
+
+ sb.Grow(120)
+ if mfr.form {
+ sb.WriteString("form-data; name=\"file")
+ if len(params) > 0 {
+ fmt.Fprintf(sb, "?%s", params.Encode())
+ }
+ sb.WriteString("\"")
+ } else {
+ sb.WriteString("attachment")
+ }
+
+ fmt.Fprintf(sb, "; filename=\"%s\"", url.QueryEscape(filename))
+
+ header.Set(contentDispositionHeader, sb.String())
+}
+
 // Boundary returns the boundary string to be used to separate files in the multipart data
 func (mfr *MultiFileReader) Boundary() string {
  return mfr.mpWriter.Boundary()