feat: Support storing UnixFS 1.5 Mode and ModTime

Replaces #7754 written by @kstuart

- ipfs/boxo#653
- ipfs/boxo#658

- [X] Can `ipfs add` with preserved mode and/or last modification time
  - [X] on files
  - [X] on directories
- [X] Can `ipfs add` with custom mode and/or last modification time
  - [X] on files
  - [X] on directories
- [X] Can `ipfs get` restoring mode and/or last modification time
  - [X] on files
  - [X] on directories
  - [X] in archives
- [X] Can `ipfs files chmod` to change mode
  - [X] on files
  - [X] on directories
- [X] Can `ipfs files touch` to change last modification time
  - [X] on files
  - [X] on directories
- [X] Automatically update the last modification time when file data is changed or truncated (e.g. `ipfs files write`)
- [X] Can add files and directories with mode and/or modification time using multipart-form data
- [X] `ipfs files stat` reports mode and last modification time

**Note:**
- [X] Adds support to `kubo/core/rpc` (may require additional tests).

- ~ipfs/interface-go-ipfs-core/pull/66~ replace by this PR
- ~ipfs/go-unixfs/pull/85~ replaced by: ipfs/boxo#658
- ~ipfs/go-mfs/pull/93~ replaced by: ipfs/boxo#658
- ~ipfs/go-ipfs-files/pull/31~ replaced by: ipfs/boxo#653
- ~ipfs/tar-utils/pull/11~ replaced by: ipfs/boxo#653

- When adding files and directories without opting to store a mode or modification time the same CIDs are generated that would have been created before this feature was implemented (opt-in).
- The Go runtime currently has no native support for restoring file mode and modification time on symbolic-links, support for restoring the last modification time has been added for Linux distributions and the following BSDs: freebsd, netbsd, openbsd, dragonflybsd.
- Automatically updating a modification time will only occur if a modification time was previously stored.
- When creating an archive, for compatibility, time resolution is to the second; Nanoseconds are not supported.

The `ipfs add` options `--preserve-mode` and `--preserve-mtime` are used to store the original mode and last modified time of the file being added, the options `--mode`, `--mtime` and `--mtime-nsecs` are used to store custom values, a custom value of 0 is a no-op as is providing `--mtime-nsecs` without `--mtime`.

The preserve flags and custom options are mutually exclusive, if both are provided the custom options take precedence.

---

Closes #6920

client/rpc/apifile.go

@@ -5,6 +5,8 @@ import (
  "encoding/json"
  "fmt"
  "io"
+ "os"
+ "time"
 
  "github.com/ipfs/boxo/files"
  unixfs "github.com/ipfs/boxo/ipld/unixfs"
@@ -24,9 +26,12 @@ func (api *UnixfsAPI) Get(ctx context.Context, p path.Path) (files.Node, error)
  }
 
  var stat struct {
- Hash string
- Type string
- Size int64 // unixfs size
+ Hash string
+ Type string
+ Size int64 // unixfs size
+ Mode os.FileMode
+ Mtime int64
+ MtimeNsecs int
  }
  err := api.core().Request("files/stat", p.String()).Exec(ctx, &stat)
  if err != nil {
@@ -35,9 +40,9 @@ func (api *UnixfsAPI) Get(ctx context.Context, p path.Path) (files.Node, error)
 
  switch stat.Type {
  case "file":
- return api.getFile(ctx, p, stat.Size)
+ return api.getFile(ctx, p, stat.Size, stat.Mode, stat.Mtime, stat.MtimeNsecs)
  case "directory":
- return api.getDir(ctx, p, stat.Size)
+ return api.getDir(ctx, p, stat.Size, stat.Mode, stat.Mtime, stat.MtimeNsecs)
  default:
  return nil, fmt.Errorf("unsupported file type '%s'", stat.Type)
  }
@@ -49,6 +54,9 @@ type apiFile struct {
  size int64
  path path.Path
 
+ mode os.FileMode
+ mtime time.Time
+
  r *Response
  at int64
 }
@@ -128,17 +136,31 @@ func (f *apiFile) Close() error {
  return nil
 }
 
+func (f *apiFile) Mode() os.FileMode {
+ return f.mode
+}
+
+func (f *apiFile) ModTime() time.Time {
+ return f.mtime
+}
+
 func (f *apiFile) Size() (int64, error) {
  return f.size, nil
 }
 
-func (api *UnixfsAPI) getFile(ctx context.Context, p path.Path, size int64) (files.Node, error) {
+func (api *UnixfsAPI) getFile(ctx context.Context, p path.Path, size int64, mode os.FileMode, mtime int64, mtimeNsecs int) (files.Node, error) {
  f := &apiFile{
  ctx: ctx,
  core: api.core(),
  size: size,
  path: p,
  }
+ if mode != 0 {
+ f.mode = os.FileMode(mode)
+ }
+ if mtime != 0 {
+ f.mtime = time.Unix(mtime, int64(mtimeNsecs)).UTC()
+ }
 
  return f, f.reset()
 }
@@ -195,13 +217,13 @@ func (it *apiIter) Next() bool {
 
  switch it.cur.Type {
  case unixfs.THAMTShard, unixfs.TMetadata, unixfs.TDirectory:
- it.curFile, err = it.core.getDir(it.ctx, path.FromCid(c), int64(it.cur.Size))
+ it.curFile, err = it.core.getDir(it.ctx, path.FromCid(c), int64(it.cur.Size), it.cur.Mode, it.cur.Mtime, it.cur.MtimeNsecs)
  if err != nil {
  it.err = err
  return false
  }
  case unixfs.TFile:
- it.curFile, err = it.core.getFile(it.ctx, path.FromCid(c), int64(it.cur.Size))
+ it.curFile, err = it.core.getFile(it.ctx, path.FromCid(c), int64(it.cur.Size), it.cur.Mode, it.cur.Mtime, it.cur.MtimeNsecs)
  if err != nil {
  it.err = err
  return false
@@ -223,13 +245,24 @@ type apiDir struct {
  size int64
  path path.Path
 
+ mode os.FileMode
+ mtime time.Time
+
  dec *json.Decoder
 }
 
 func (d *apiDir) Close() error {
  return nil
 }
 
+func (d *apiDir) Mode() os.FileMode {
+ return d.mode
+}
+
+func (d *apiDir) ModTime() time.Time {
+ return d.mtime
+}
+
 func (d *apiDir) Size() (int64, error) {
  return d.size, nil
 }
@@ -242,7 +275,7 @@ func (d *apiDir) Entries() files.DirIterator {
  }
 }
 
-func (api *UnixfsAPI) getDir(ctx context.Context, p path.Path, size int64) (files.Node, error) {
+func (api *UnixfsAPI) getDir(ctx context.Context, p path.Path, size int64, mode os.FileMode, mtime int64, mtimeNsecs int) (files.Node, error) {
  resp, err := api.core().Request("ls", p.String()).
  Option("resolve-size", true).
  Option("stream", true).Send(ctx)
@@ -262,6 +295,13 @@ func (api *UnixfsAPI) getDir(ctx context.Context, p path.Path, size int64) (file
  dec: json.NewDecoder(resp.Output),
  }
 
+ if mode != 0 {
+ d.mode = os.FileMode(mode)
+ }
+ if mtime != 0 {
+ d.mtime = time.Unix(mtime, int64(mtimeNsecs)).UTC()
+ }
+
  return d, nil
 }
 

client/rpc/unixfs.go

@@ -6,6 +6,7 @@ import (
  "errors"
  "fmt"
  "io"
+ "os"
 
  "github.com/ipfs/boxo/files"
  unixfs "github.com/ipfs/boxo/ipld/unixfs"
@@ -22,6 +23,10 @@ type addEvent struct {
  Hash string `json:",omitempty"`
  Bytes int64 `json:",omitempty"`
  Size string `json:",omitempty"`
+
+ Mode os.FileMode `json:",omitempty"`
+ Mtime int64 `json:",omitempty"`
+ MtimeNsecs int `json:",omitempty"`
 }
 
 type UnixfsAPI HttpApi
@@ -94,9 +99,12 @@ loop:
 
  if options.Events != nil {
  ifevt := &iface.AddEvent{
- Name: out.Name,
- Size: out.Size,
- Bytes: out.Bytes,
+ Name: out.Name,
+ Size: out.Size,
+ Bytes: out.Bytes,
+ Mode: out.Mode,
+ Mtime: out.Mtime,
+ MtimeNsecs: out.MtimeNsecs,
  }
 
  if out.Hash != "" {
@@ -129,6 +137,10 @@ type lsLink struct {
  Size uint64
  Type unixfs_pb.Data_DataType
  Target string
+
+ Mode os.FileMode
+ Mtime int64
+ MtimeNsecs int
 }
 
 type lsObject struct {

core/commands/add.go

@@ -6,7 +6,9 @@ import (
  "io"
  "os"
  gopath "path"
+ "strconv"
  "strings"
+ "time"
 
  "github.com/ipfs/kubo/config"
  "github.com/ipfs/kubo/core/commands/cmdenv"
@@ -25,11 +27,31 @@ import (
 // ErrDepthLimitExceeded indicates that the max depth has been exceeded.
 var ErrDepthLimitExceeded = fmt.Errorf("depth limit exceeded")
 
+type TimeParts struct {
+ t *time.Time
+}
+
+func (t TimeParts) MarshalJSON() ([]byte, error) {
+ return t.t.MarshalJSON()
+}
+
+// UnmarshalJSON implements the json.Unmarshaler interface.
+// The time is expected to be a quoted string in RFC 3339 format.
+func (t *TimeParts) UnmarshalJSON(data []byte) (err error) {
+ // Fractional seconds are handled implicitly by Parse.
+ tt, err := time.Parse("\"2006-01-02T15:04:05Z\"", string(data))
+ *t = TimeParts{&tt}
+ return
+}
+
 type AddEvent struct {
- Name string
- Hash string `json:",omitempty"`
- Bytes int64 `json:",omitempty"`
- Size string `json:",omitempty"`
+ Name string
+ Hash string `json:",omitempty"`
+ Bytes int64 `json:",omitempty"`
+ Size string `json:",omitempty"`
+ Mode string `json:",omitempty"`
+ Mtime int64 `json:",omitempty"`
+ MtimeNsecs int `json:",omitempty"`
 }
 
 const (
@@ -50,6 +72,12 @@ const (
  inlineOptionName = "inline"
  inlineLimitOptionName = "inline-limit"
  toFilesOptionName = "to-files"
+
+ preserveModeOptionName = "preserve-mode"
+ preserveMtimeOptionName = "preserve-mtime"
+ modeOptionName = "mode"
+ mtimeOptionName = "mtime"
+ mtimeNsecsOptionName = "mtime-nsecs"
 )
 
 const adderOutChanSize = 8
@@ -166,6 +194,12 @@ See 'dag export' and 'dag import' for more information.
  cmds.IntOption(inlineLimitOptionName, "Maximum block size to inline. (experimental)").WithDefault(32),
  cmds.BoolOption(pinOptionName, "Pin locally to protect added files from garbage collection.").WithDefault(true),
  cmds.StringOption(toFilesOptionName, "Add reference to Files API (MFS) at the provided path."),
+
+ cmds.BoolOption(preserveModeOptionName, "Apply existing POSIX permissions to created UnixFS entries"),
+ cmds.BoolOption(preserveMtimeOptionName, "Apply existing POSIX modification time to created UnixFS entries"),
+ cmds.UintOption(modeOptionName, "Custom POSIX file mode to store in created UnixFS entries"),
+ cmds.Int64Option(mtimeOptionName, "Custom POSIX modification time to store in created UnixFS entries (seconds before or after the Unix Epoch)"),
+ cmds.UintOption(mtimeNsecsOptionName, "Custom POSIX modification time (optional time fraction in nanoseconds)"),
  },
  PreRun: func(req *cmds.Request, env cmds.Environment) error {
  quiet, _ := req.Options[quietOptionName].(bool)
@@ -217,6 +251,11 @@ See 'dag export' and 'dag import' for more information.
  inline, _ := req.Options[inlineOptionName].(bool)
  inlineLimit, _ := req.Options[inlineLimitOptionName].(int)
  toFilesStr, toFilesSet := req.Options[toFilesOptionName].(string)
+ preserveMode, _ := req.Options[preserveModeOptionName].(bool)
+ preserveMtime, _ := req.Options[preserveMtimeOptionName].(bool)
+ mode, _ := req.Options[modeOptionName].(uint)
+ mtime, _ := req.Options[mtimeOptionName].(int64)
+ mtimeNsecs, _ := req.Options[mtimeNsecsOptionName].(uint)
 
  if chunker == "" {
  chunker = cfg.Import.UnixFSChunker.WithDefault(config.DefaultUnixFSChunker)
@@ -272,6 +311,19 @@ See 'dag export' and 'dag import' for more information.
 
  options.Unixfs.Progress(progress),
  options.Unixfs.Silent(silent),
+
+ options.Unixfs.PreserveMode(preserveMode),
+ options.Unixfs.PreserveMtime(preserveMtime),
+ }
+
+ if mode != 0 {
+ opts = append(opts, options.Unixfs.Mode(os.FileMode(mode)))
+ }
+
+ if mtime != 0 {
+ opts = append(opts, options.Unixfs.Mtime(mtime, uint32(mtimeNsecs)))
+ } else if mtimeNsecs != 0 {
+ fmt.Println("option", mtimeNsecsOptionName, "ignored as no valid", mtimeOptionName, "value provided")
  }
 
  if cidVerSet {
@@ -383,12 +435,33 @@ See 'dag export' and 'dag import' for more information.
  output.Name = gopath.Join(addit.Name(), output.Name)
  }
 
- if err := res.Emit(&AddEvent{
- Name: output.Name,
- Hash: h,
- Bytes: output.Bytes,
- Size: output.Size,
- }); err != nil {
+ output.Mode = addit.Node().Mode()
+ if ts := addit.Node().ModTime(); !ts.IsZero() {
+ output.Mtime = addit.Node().ModTime().Unix()
+ output.MtimeNsecs = addit.Node().ModTime().Nanosecond()
+ }
+
+ addEvent := AddEvent{
+ Name: output.Name,
+ Hash: h,
+ Bytes: output.Bytes,
+ Size: output.Size,
+ Mtime: output.Mtime,
+ MtimeNsecs: output.MtimeNsecs,
+ }
+
+ if output.Mode != 0 {
+ addEvent.Mode = "0" + strconv.FormatUint(uint64(output.Mode), 8)
+ }
+
+ if output.Mtime > 0 {
+ addEvent.Mtime = output.Mtime
+ if output.MtimeNsecs > 0 {
+ addEvent.MtimeNsecs = output.MtimeNsecs
+ }
+ }
+
+ if err := res.Emit(&addEvent); err != nil {
  return err
  }
  }

core/commands/commands_test.go

@@ -89,6 +89,8 @@ func TestCommands(t *testing.T) {
  "/files/rm",
  "/files/stat",
  "/files/write",
+ "/files/chmod",
+ "/files/touch",
  "/filestore",
  "/filestore/dups",
  "/filestore/ls",