From 0be5dde803a9154698dcfbaf8727b313b6854365 Mon Sep 17 00:00:00 2001 From: Ian Davis Date: Wed, 15 Dec 2021 09:40:55 +0000 Subject: [PATCH 1/2] Support unixfs 1.5 inline metadata --- importer/balanced/balanced_test.go | 96 +++++++++++++++++++++- importer/balanced/builder.go | 32 +++++--- importer/helpers/dagbuilder.go | 56 ++++++++++++- importer/trickle/trickle_test.go | 110 +++++++++++++++++++++++++- importer/trickle/trickledag.go | 1 + io/dagreader.go | 29 ++++++- pb/unixfs.pb.go | 123 ++++++++++++++++++++++------- pb/unixfs.proto | 9 +++ unixfs.go | 83 ++++++++++++++++++- unixfs_test.go | 108 ++++++++++++++++++++++++- 10 files changed, 595 insertions(+), 52 deletions(-) diff --git a/importer/balanced/balanced_test.go b/importer/balanced/balanced_test.go index b2069e3a9..ea27a61bd 100644 --- a/importer/balanced/balanced_test.go +++ b/importer/balanced/balanced_test.go @@ -7,12 +7,15 @@ import ( "io" "io/ioutil" mrand "math/rand" + "os" "testing" + "time" h "github.com/ipfs/go-unixfs/importer/helpers" uio "github.com/ipfs/go-unixfs/io" chunker "github.com/ipfs/go-ipfs-chunker" + files "github.com/ipfs/go-ipfs-files" u "github.com/ipfs/go-ipfs-util" ipld "github.com/ipfs/go-ipld-format" dag "github.com/ipfs/go-merkledag" @@ -27,6 +30,10 @@ func buildTestDag(ds ipld.DAGService, spl chunker.Splitter) (*dag.ProtoNode, err Maxlinks: h.DefaultLinksPerBlock, } + return buildTestDagWithParams(ds, spl, dbp) +} + +func buildTestDagWithParams(ds ipld.DAGService, spl chunker.Splitter, dbp h.DagBuilderParams) (*dag.ProtoNode, error) { db, err := dbp.New(spl) if err != nil { return nil, err @@ -53,7 +60,7 @@ func getTestDag(t *testing.T, ds ipld.DAGService, size int64, blksize int64) (*d return nd, data } -//Test where calls to read are smaller than the chunk size +// Test where calls to read are smaller than the chunk size func TestSizeBasedSplit(t *testing.T) { if testing.Short() { t.SkipNow() @@ -299,7 +306,6 @@ func TestSeekingStress(t *testing.T) { t.Fatal(err) } } - } func TestSeekingConsistency(t *testing.T) { @@ -337,3 +343,89 @@ func TestSeekingConsistency(t *testing.T) { t.Fatal(err) } } + +func TestMetadata(t *testing.T) { + nbytes := 3 * chunker.DefaultBlockSize + buf := new(bytes.Buffer) + io.CopyN(buf, u.NewTimeSeededRand(), int64(nbytes)) + + dagserv := mdtest.Mock() + dbp := h.DagBuilderParams{ + Dagserv: dagserv, + Maxlinks: h.DefaultLinksPerBlock, + FileMode: 0o522, + ModTime: time.Unix(1638111600, 76552), + } + + nd, err := buildTestDagWithParams(dagserv, chunker.DefaultSplitter(buf), dbp) + if err != nil { + t.Fatal(err) + } + + dr, err := uio.NewDagReader(context.Background(), nd, dagserv) + if err != nil { + t.Fatal(err) + } + + if !dr.ModTime().Equal(dbp.ModTime) { + t.Errorf("got modtime %v, wanted %v", dr.ModTime(), dbp.ModTime) + } + + if dr.FileMode() != dbp.FileMode { + t.Errorf("got filemode %o, wanted %o", dr.FileMode(), dbp.FileMode) + } +} + +type fileinfo struct { + name string + size int64 + mode os.FileMode + mtime time.Time +} + +func (fi *fileinfo) Name() string { return fi.name } +func (fi *fileinfo) Size() int64 { return fi.size } +func (fi *fileinfo) Mode() os.FileMode { return fi.mode } +func (fi *fileinfo) ModTime() time.Time { return fi.mtime } +func (fi *fileinfo) IsDir() bool { return false } +func (fi *fileinfo) Sys() interface{} { return nil } + +func TestMetadataFromFilestore(t *testing.T) { + nbytes := 3 * chunker.DefaultBlockSize + buf := new(bytes.Buffer) + io.CopyN(buf, u.NewTimeSeededRand(), int64(nbytes)) + + fi := &fileinfo{ + mode: 0o522, + mtime: time.Unix(1638111600, 76552), + } + + rpf, err := files.NewReaderPathFile("/path", ioutil.NopCloser(buf), fi) + if err != nil { + t.Fatalf("new reader path file: %v", err) + } + + dagserv := mdtest.Mock() + dbp := h.DagBuilderParams{ + Dagserv: dagserv, + Maxlinks: h.DefaultLinksPerBlock, + NoCopy: true, + } + nd, err := buildTestDagWithParams(dagserv, chunker.DefaultSplitter(rpf), dbp) + if err != nil { + t.Fatal(err) + } + + dr, err := uio.NewDagReader(context.Background(), nd, dagserv) + if err != nil { + t.Fatal(err) + } + + if !dr.ModTime().Equal(fi.mtime) { + t.Errorf("got modtime %v, wanted %v", dr.ModTime(), fi.mtime) + } + + if dr.FileMode() != fi.mode { + t.Errorf("got filemode %o, wanted %o", dr.FileMode(), fi.mode) + } +} diff --git a/importer/balanced/builder.go b/importer/balanced/builder.go index 407117dad..d99760e7f 100644 --- a/importer/balanced/builder.go +++ b/importer/balanced/builder.go @@ -163,10 +163,22 @@ func Layout(db *h.DagBuilderHelper) (ipld.Node, error) { // Fill the `newRoot` (that has the old `root` already as child) // and make it the current `root` for the next iteration (when // it will become "old"). - root, fileSize, err = fillNodeRec(db, newRoot, depth) + var potentialRoot *h.FSNodeOverDag + potentialRoot, fileSize, err = fillNodeRec(db, newRoot, depth) if err != nil { return nil, err } + + // Only add file metadata to the top level root + if db.Done() { + db.FillMetadata(potentialRoot) + } + + root, err = potentialRoot.Commit() + if err != nil { + return nil, err + } + } return root, db.Add(root) @@ -212,7 +224,7 @@ func Layout(db *h.DagBuilderHelper) (ipld.Node, error) { // seeking through the DAG when reading data later). // // warning: **children** pinned indirectly, but input node IS NOT pinned. -func fillNodeRec(db *h.DagBuilderHelper, node *h.FSNodeOverDag, depth int) (filledNode ipld.Node, nodeFileSize uint64, err error) { +func fillNodeRec(db *h.DagBuilderHelper, node *h.FSNodeOverDag, depth int) (filledNode *h.FSNodeOverDag, nodeFileSize uint64, err error) { if depth < 1 { return nil, 0, errors.New("attempt to fillNode at depth < 1") } @@ -240,10 +252,16 @@ func fillNodeRec(db *h.DagBuilderHelper, node *h.FSNodeOverDag, depth int) (fill } else { // Recursion case: create an internal node to in turn keep // descending in the DAG and adding child nodes to it. - childNode, childFileSize, err = fillNodeRec(db, nil, depth-1) + var internalNode *h.FSNodeOverDag + internalNode, childFileSize, err = fillNodeRec(db, nil, depth-1) if err != nil { return nil, 0, err } + childNode, err = internalNode.Commit() + if err != nil { + return nil, 0, err + } + } err = node.AddChild(childNode, childFileSize, db) @@ -254,11 +272,5 @@ func fillNodeRec(db *h.DagBuilderHelper, node *h.FSNodeOverDag, depth int) (fill nodeFileSize = node.FileSize() - // Get the final `dag.ProtoNode` with the `FSNode` data encoded inside. - filledNode, err = node.Commit() - if err != nil { - return nil, 0, err - } - - return filledNode, nodeFileSize, nil + return node, nodeFileSize, nil } diff --git a/importer/helpers/dagbuilder.go b/importer/helpers/dagbuilder.go index e3cf7b44f..8f484bb68 100644 --- a/importer/helpers/dagbuilder.go +++ b/importer/helpers/dagbuilder.go @@ -5,6 +5,7 @@ import ( "errors" "io" "os" + "time" dag "github.com/ipfs/go-merkledag" @@ -30,6 +31,8 @@ type DagBuilderHelper struct { nextData []byte // the next item to return. maxlinks int cidBuilder cid.Builder + modTime time.Time + fileMode os.FileMode // Filestore support variables. // ---------------------------- @@ -65,6 +68,14 @@ type DagBuilderParams struct { // NoCopy signals to the chunker that it should track fileinfo for // filestore adds NoCopy bool + + // ModTime is the optional file modification tiome to be embedded in the final dag + // Note that this will be overwritten by the fileinfo of the underlying filestore file if NoCopy is true + ModTime time.Time + + // FileMode is the optional file mode metadata to be embedded in the final dag + // Note that this will be overwritten by the fileinfo of the underlying filestore file if NoCopy is true + FileMode os.FileMode } // New generates a new DagBuilderHelper from the given params and a given @@ -76,10 +87,16 @@ func (dbp *DagBuilderParams) New(spl chunker.Splitter) (*DagBuilderHelper, error rawLeaves: dbp.RawLeaves, cidBuilder: dbp.CidBuilder, maxlinks: dbp.Maxlinks, + modTime: dbp.ModTime, + fileMode: dbp.FileMode, } if fi, ok := spl.Reader().(files.FileInfo); dbp.NoCopy && ok { db.fullPath = fi.AbsPath() db.stat = fi.Stat() + if db.stat != nil { + db.modTime = db.stat.ModTime() + db.fileMode = db.stat.Mode() + } } if dbp.NoCopy && db.fullPath == "" { // Enforce NoCopy @@ -177,7 +194,6 @@ func (db *DagBuilderHelper) NewLeafNode(data []byte, fsNodeType pb.Data_DataType // NOTE: This function creates raw data nodes so it only works // for the `trickle.Layout`. func (db *DagBuilderHelper) FillNodeLayer(node *FSNodeOverDag) error { - // while we have room AND we're not done for node.NumChildren() < db.maxlinks && !db.Done() { child, childFileSize, err := db.NewLeafDataNode(ft.TRaw) @@ -232,7 +248,7 @@ func (db *DagBuilderHelper) NewLeafDataNode(fsNodeType pb.Data_DataType) (node i // offset is more related to this function). func (db *DagBuilderHelper) ProcessFileStore(node ipld.Node, dataSize uint64) ipld.Node { // Check if Filestore is being used. - if db.fullPath != "" { + if db.isUsingFilestore() { // Check if the node is actually a raw node (needed for // Filestore support). if _, ok := node.(*dag.RawNode); ok { @@ -267,6 +283,18 @@ func (db *DagBuilderHelper) Maxlinks() int { return db.maxlinks } +func (db *DagBuilderHelper) isUsingFilestore() bool { + return db.fullPath != "" +} + +// FillMetadata sets metadata attributes on the supplied node. +func (db *DagBuilderHelper) FillMetadata(node *FSNodeOverDag) error { + node.SetFileMode(db.fileMode) + node.SetModTime(db.modTime) + + return nil +} + // FSNodeOverDag encapsulates an `unixfs.FSNode` that will be stored in a // `dag.ProtoNode`. Instead of just having a single `ipld.Node` that // would need to be constantly (un)packed to access and modify its @@ -398,3 +426,27 @@ func (n *FSNodeOverDag) GetChild(ctx context.Context, i int, ds ipld.DAGService) return NewFSNFromDag(pbn) } + +// FileMode returns the file mode bits from the underlying +// representation of the `ft.FSNode`. +func (n *FSNodeOverDag) FileMode() os.FileMode { + return n.file.FileMode() +} + +// SetFileMode sets the file mode bits in the underlying +// representation of the `ft.FSNode`. +func (n *FSNodeOverDag) SetFileMode(m os.FileMode) { + n.file.SetFileMode(m) +} + +// ModTime returns the modification time of the file from the underlying +// representation of the `ft.FSNode`. +func (n *FSNodeOverDag) ModTime() time.Time { + return n.file.ModTime() +} + +// SetModTime sets the modification time of the file in the underlying +// representation of the `ft.FSNode`. +func (n *FSNodeOverDag) SetModTime(t time.Time) { + n.file.SetModTime(t) +} diff --git a/importer/trickle/trickle_test.go b/importer/trickle/trickle_test.go index 2b6e0bd46..7b4da883a 100644 --- a/importer/trickle/trickle_test.go +++ b/importer/trickle/trickle_test.go @@ -7,13 +7,16 @@ import ( "io" "io/ioutil" mrand "math/rand" + "os" "testing" + "time" ft "github.com/ipfs/go-unixfs" h "github.com/ipfs/go-unixfs/importer/helpers" uio "github.com/ipfs/go-unixfs/io" chunker "github.com/ipfs/go-ipfs-chunker" + files "github.com/ipfs/go-ipfs-files" u "github.com/ipfs/go-ipfs-util" ipld "github.com/ipfs/go-ipld-format" merkledag "github.com/ipfs/go-merkledag" @@ -39,6 +42,10 @@ func buildTestDag(ds ipld.DAGService, spl chunker.Splitter, rawLeaves UseRawLeav RawLeaves: bool(rawLeaves), } + return buildTestDagWithParams(ds, spl, dbp) +} + +func buildTestDagWithParams(ds ipld.DAGService, spl chunker.Splitter, dbp h.DagBuilderParams) (*merkledag.ProtoNode, error) { db, err := dbp.New(spl) if err != nil { return nil, err @@ -58,11 +65,11 @@ func buildTestDag(ds ipld.DAGService, spl chunker.Splitter, rawLeaves UseRawLeav Getter: ds, Direct: dbp.Maxlinks, LayerRepeat: depthRepeat, - RawLeaves: bool(rawLeaves), + RawLeaves: dbp.RawLeaves, }) } -//Test where calls to read are smaller than the chunk size +// Test where calls to read are smaller than the chunk size func TestSizeBasedSplit(t *testing.T) { runBothSubtests(t, testSizeBasedSplit) } @@ -432,7 +439,6 @@ func testSeekingStress(t *testing.T, rawLeaves UseRawLeaves) { t.Fatal(err) } } - } func TestSeekingConsistency(t *testing.T) { @@ -665,3 +671,101 @@ func TestAppendSingleBytesToEmpty(t *testing.T) { t.Fatal(err) } } + +func TestMetadata(t *testing.T) { + runBothSubtests(t, testMetadata) +} + +func testMetadata(t *testing.T, rawLeaves UseRawLeaves) { + nbytes := 3 * chunker.DefaultBlockSize + buf := new(bytes.Buffer) + io.CopyN(buf, u.NewTimeSeededRand(), int64(nbytes)) + + dagserv := mdtest.Mock() + dbp := h.DagBuilderParams{ + Dagserv: dagserv, + Maxlinks: h.DefaultLinksPerBlock, + RawLeaves: bool(rawLeaves), + FileMode: 0o522, + ModTime: time.Unix(1638111600, 76552), + } + + nd, err := buildTestDagWithParams(dagserv, chunker.DefaultSplitter(buf), dbp) + if err != nil { + t.Fatal(err) + } + + dr, err := uio.NewDagReader(context.Background(), nd, dagserv) + if err != nil { + t.Fatal(err) + } + + if !dr.ModTime().Equal(dbp.ModTime) { + t.Errorf("got modtime %v, wanted %v", dr.ModTime(), dbp.ModTime) + } + + if dr.FileMode() != dbp.FileMode { + t.Errorf("got filemode %o, wanted %o", dr.FileMode(), dbp.FileMode) + } +} + +func TestMetadataFromFilestore(t *testing.T) { + runBothSubtests(t, testMetadataFromFilestore) +} + +type fileinfo struct { + name string + size int64 + mode os.FileMode + mtime time.Time +} + +func (fi *fileinfo) Name() string { return fi.name } +func (fi *fileinfo) Size() int64 { return fi.size } +func (fi *fileinfo) Mode() os.FileMode { return fi.mode } +func (fi *fileinfo) ModTime() time.Time { return fi.mtime } +func (fi *fileinfo) IsDir() bool { return false } +func (fi *fileinfo) Sys() interface{} { return nil } + +func testMetadataFromFilestore(t *testing.T, rawLeaves UseRawLeaves) { + nbytes := 3 * chunker.DefaultBlockSize + buf := new(bytes.Buffer) + io.CopyN(buf, u.NewTimeSeededRand(), int64(nbytes)) + + fi := &fileinfo{ + mode: 0o522, + mtime: time.Unix(1638111600, 76552), + } + + rpf, err := files.NewReaderPathFile("/path", ioutil.NopCloser(buf), fi) + if err != nil { + t.Fatalf("new reader path file: %v", err) + } + + dagserv := mdtest.Mock() + + dbp := h.DagBuilderParams{ + Dagserv: dagserv, + Maxlinks: h.DefaultLinksPerBlock, + RawLeaves: bool(rawLeaves), + NoCopy: true, + } + + nd, err := buildTestDagWithParams(dagserv, chunker.DefaultSplitter(rpf), dbp) + if err != nil { + t.Fatal(err) + } + + dr, err := uio.NewDagReader(context.Background(), nd, dagserv) + if err != nil { + t.Fatal(err) + } + + if !dr.ModTime().Equal(fi.mtime) { + t.Errorf("got modtime %v, wanted %v", dr.ModTime(), fi.mtime) + } + + if dr.FileMode() != fi.mode { + t.Errorf("got filemode %o, wanted %o", dr.FileMode(), fi.mode) + } +} diff --git a/importer/trickle/trickledag.go b/importer/trickle/trickledag.go index 3a631adb8..c3f01a8db 100644 --- a/importer/trickle/trickledag.go +++ b/importer/trickle/trickledag.go @@ -38,6 +38,7 @@ const depthRepeat = 4 // explanation. func Layout(db *h.DagBuilderHelper) (ipld.Node, error) { newRoot := db.NewFSNodeOverDag(ft.TFile) + db.FillMetadata(newRoot) root, _, err := fillTrickleRec(db, newRoot, -1) if err != nil { return nil, err diff --git a/io/dagreader.go b/io/dagreader.go index 374b50916..b15fc966c 100644 --- a/io/dagreader.go +++ b/io/dagreader.go @@ -5,6 +5,8 @@ import ( "context" "errors" "io" + "os" + "time" ipld "github.com/ipfs/go-ipld-format" mdag "github.com/ipfs/go-merkledag" @@ -29,6 +31,8 @@ var ( type DagReader interface { ReadSeekCloser Size() uint64 + FileMode() os.FileMode + ModTime() time.Time CtxReadFull(context.Context, []byte) (int, error) } @@ -44,6 +48,8 @@ type ReadSeekCloser interface { // the given node, using the passed in DAGService for data retrieval. func NewDagReader(ctx context.Context, n ipld.Node, serv ipld.NodeGetter) (DagReader, error) { var size uint64 + var mode os.FileMode + var mtime time.Time switch n := n.(type) { case *mdag.RawNode: @@ -58,6 +64,8 @@ func NewDagReader(ctx context.Context, n ipld.Node, serv ipld.NodeGetter) (DagRe switch fsNode.Type() { case unixfs.TFile, unixfs.TRaw: size = fsNode.FileSize() + mode = fsNode.FileMode() + mtime = fsNode.ModTime() case unixfs.TDirectory, unixfs.THAMTShard: // Dont allow reading directories @@ -93,6 +101,8 @@ func NewDagReader(ctx context.Context, n ipld.Node, serv ipld.NodeGetter) (DagRe cancel: cancel, serv: serv, size: size, + mode: mode, + mtime: mtime, rootNode: n, dagWalker: ipld.NewWalker(ctxWithCancel, ipld.NewNavigableIPLDNode(n, serv)), }, nil @@ -115,6 +125,12 @@ type dagReader struct { // Implements the `Size()` API. size uint64 + // mode is the optional file mode bits metadata from the unixfs dag + mode os.FileMode + + // mtime is the optional modification time metadata from the unixfs dag + mtime time.Time + // Current offset for the read head within the DAG file. offset int64 @@ -138,6 +154,16 @@ func (dr *dagReader) Size() uint64 { return dr.size } +// FileMode returns the optional file mode bits +func (dr *dagReader) FileMode() os.FileMode { + return dr.mode +} + +// ModTime returns the optional modification time of the file. +func (dr *dagReader) ModTime() time.Time { + return dr.mtime +} + // Read implements the `io.Reader` interface through the `CtxReadFull` // method using the DAG reader's internal context. func (dr *dagReader) Read(b []byte) (int, error) { @@ -227,7 +253,6 @@ func (dr *dagReader) saveNodeData(node ipld.Node) error { // any errors as it's always reading from a `bytes.Reader` and asking only // the available data in it. func (dr *dagReader) readNodeDataBuffer(out []byte) int { - n, _ := dr.currentNodeData.Read(out) // Ignore the error as the EOF may not be returned in the first // `Read` call, explicitly ask for an empty buffer below to check @@ -253,7 +278,6 @@ func (dr *dagReader) readNodeDataBuffer(out []byte) int { // TODO: Check what part of the logic between the two functions // can be extracted away. func (dr *dagReader) writeNodeDataBuffer(w io.Writer) (int64, error) { - n, err := dr.currentNodeData.WriteTo(w) if err != nil { return n, err @@ -450,7 +474,6 @@ func (dr *dagReader) Seek(offset int64, whence int) (int64, error) { // In the leaf node case the search will stop here. } }) - if err != nil { return 0, err } diff --git a/pb/unixfs.pb.go b/pb/unixfs.pb.go index 6f1c8fe83..2dabe9cb5 100644 --- a/pb/unixfs.pb.go +++ b/pb/unixfs.pb.go @@ -18,7 +18,7 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package type Data_DataType int32 @@ -73,15 +73,18 @@ func (Data_DataType) EnumDescriptor() ([]byte, []int) { } type Data struct { - Type *Data_DataType `protobuf:"varint,1,req,name=Type,enum=unixfs.pb.Data_DataType" json:"Type,omitempty"` - Data []byte `protobuf:"bytes,2,opt,name=Data" json:"Data,omitempty"` - Filesize *uint64 `protobuf:"varint,3,opt,name=filesize" json:"filesize,omitempty"` - Blocksizes []uint64 `protobuf:"varint,4,rep,name=blocksizes" json:"blocksizes,omitempty"` - HashType *uint64 `protobuf:"varint,5,opt,name=hashType" json:"hashType,omitempty"` - Fanout *uint64 `protobuf:"varint,6,opt,name=fanout" json:"fanout,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Type *Data_DataType `protobuf:"varint,1,req,name=Type,enum=unixfs.pb.Data_DataType" json:"Type,omitempty"` + Data []byte `protobuf:"bytes,2,opt,name=Data" json:"Data,omitempty"` + Filesize *uint64 `protobuf:"varint,3,opt,name=filesize" json:"filesize,omitempty"` + Blocksizes []uint64 `protobuf:"varint,4,rep,name=blocksizes" json:"blocksizes,omitempty"` + HashType *uint64 `protobuf:"varint,5,opt,name=hashType" json:"hashType,omitempty"` + Fanout *uint64 `protobuf:"varint,6,opt,name=fanout" json:"fanout,omitempty"` + // unixfs v1.5 metadata additions + Mode *uint32 `protobuf:"varint,7,opt,name=mode" json:"mode,omitempty"` + Mtime *UnixTime `protobuf:"bytes,8,opt,name=mtime" json:"mtime,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *Data) Reset() { *m = Data{} } @@ -150,6 +153,20 @@ func (m *Data) GetFanout() uint64 { return 0 } +func (m *Data) GetMode() uint32 { + if m != nil && m.Mode != nil { + return *m.Mode + } + return 0 +} + +func (m *Data) GetMtime() *UnixTime { + if m != nil { + return m.Mtime + } + return nil +} + type Metadata struct { MimeType *string `protobuf:"bytes,1,opt,name=MimeType" json:"MimeType,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -188,30 +205,82 @@ func (m *Metadata) GetMimeType() string { return "" } +type UnixTime struct { + Seconds *int64 `protobuf:"varint,1,req,name=Seconds" json:"Seconds,omitempty"` + FractionalNanoseconds *uint32 `protobuf:"fixed32,2,opt,name=FractionalNanoseconds" json:"FractionalNanoseconds,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *UnixTime) Reset() { *m = UnixTime{} } +func (m *UnixTime) String() string { return proto.CompactTextString(m) } +func (*UnixTime) ProtoMessage() {} +func (*UnixTime) Descriptor() ([]byte, []int) { + return fileDescriptor_e2fd76cc44dfc7c3, []int{2} +} +func (m *UnixTime) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_UnixTime.Unmarshal(m, b) +} +func (m *UnixTime) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_UnixTime.Marshal(b, m, deterministic) +} +func (m *UnixTime) XXX_Merge(src proto.Message) { + xxx_messageInfo_UnixTime.Merge(m, src) +} +func (m *UnixTime) XXX_Size() int { + return xxx_messageInfo_UnixTime.Size(m) +} +func (m *UnixTime) XXX_DiscardUnknown() { + xxx_messageInfo_UnixTime.DiscardUnknown(m) +} + +var xxx_messageInfo_UnixTime proto.InternalMessageInfo + +func (m *UnixTime) GetSeconds() int64 { + if m != nil && m.Seconds != nil { + return *m.Seconds + } + return 0 +} + +func (m *UnixTime) GetFractionalNanoseconds() uint32 { + if m != nil && m.FractionalNanoseconds != nil { + return *m.FractionalNanoseconds + } + return 0 +} + func init() { proto.RegisterEnum("unixfs.pb.Data_DataType", Data_DataType_name, Data_DataType_value) proto.RegisterType((*Data)(nil), "unixfs.pb.Data") proto.RegisterType((*Metadata)(nil), "unixfs.pb.Metadata") + proto.RegisterType((*UnixTime)(nil), "unixfs.pb.UnixTime") } func init() { proto.RegisterFile("unixfs.proto", fileDescriptor_e2fd76cc44dfc7c3) } var fileDescriptor_e2fd76cc44dfc7c3 = []byte{ - // 254 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x4c, 0x90, 0xb1, 0x6a, 0xeb, 0x30, - 0x18, 0x85, 0xaf, 0x6c, 0x25, 0xb1, 0xff, 0xeb, 0x16, 0xf1, 0x0f, 0x45, 0x74, 0x28, 0xc6, 0x43, - 0xd1, 0x50, 0x3c, 0xf4, 0x0d, 0x0a, 0xa1, 0x74, 0xf1, 0xa2, 0x84, 0xee, 0x4a, 0x22, 0x63, 0x11, - 0xc7, 0x0a, 0xb6, 0x42, 0xeb, 0x3e, 0x45, 0x1f, 0xb9, 0xc8, 0x8e, 0xdd, 0x2e, 0x82, 0x4f, 0xe7, - 0x7c, 0xe2, 0x20, 0x48, 0x2e, 0x8d, 0xf9, 0x2c, 0xbb, 0xfc, 0xdc, 0x5a, 0x67, 0x31, 0x9e, 0x68, - 0x97, 0x7d, 0x07, 0x40, 0xd7, 0xca, 0x29, 0x7c, 0x02, 0xba, 0xed, 0xcf, 0x9a, 0x93, 0x34, 0x10, - 0xb7, 0xcf, 0x3c, 0x9f, 0x2b, 0xb9, 0x8f, 0x87, 0xc3, 0xe7, 0x72, 0x68, 0x21, 0x8e, 0x16, 0x0f, - 0x52, 0x22, 0x12, 0x39, 0xbe, 0x70, 0x0f, 0x51, 0x69, 0x6a, 0xdd, 0x99, 0x2f, 0xcd, 0xc3, 0x94, - 0x08, 0x2a, 0x67, 0xc6, 0x07, 0x80, 0x5d, 0x6d, 0xf7, 0x47, 0x0f, 0x1d, 0xa7, 0x69, 0x28, 0xa8, - 0xfc, 0x73, 0xe3, 0xdd, 0x4a, 0x75, 0xd5, 0xb0, 0x60, 0x31, 0xba, 0x13, 0xe3, 0x1d, 0x2c, 0x4b, - 0xd5, 0xd8, 0x8b, 0xe3, 0xcb, 0x21, 0xb9, 0x52, 0xf6, 0x0e, 0xd1, 0xb4, 0x0a, 0x57, 0x10, 0x4a, - 0xf5, 0xc1, 0xfe, 0xe1, 0x0d, 0xc4, 0x6b, 0xd3, 0xea, 0xbd, 0xb3, 0x6d, 0xcf, 0x08, 0x46, 0x40, - 0x5f, 0x4d, 0xad, 0x59, 0x80, 0x09, 0x44, 0x85, 0x76, 0xea, 0xa0, 0x9c, 0x62, 0x21, 0xfe, 0x87, - 0xd5, 0xa6, 0x3f, 0xd5, 0xa6, 0x39, 0x32, 0xea, 0x9d, 0xb7, 0x97, 0x62, 0xbb, 0xa9, 0x54, 0x7b, - 0x60, 0x8b, 0xec, 0xf1, 0xb7, 0xe9, 0x77, 0x15, 0xe6, 0xa4, 0xaf, 0x3f, 0x43, 0x44, 0x2c, 0x67, - 0xfe, 0x09, 0x00, 0x00, 0xff, 0xff, 0xe9, 0xa0, 0x51, 0x10, 0x54, 0x01, 0x00, 0x00, + // 335 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x90, 0x41, 0x6f, 0xe2, 0x30, + 0x10, 0x85, 0x37, 0x89, 0x21, 0x61, 0x80, 0x55, 0x34, 0xab, 0x5d, 0x59, 0x7b, 0xa8, 0xa2, 0x1c, + 0x2a, 0x57, 0xaa, 0x38, 0xa0, 0xfe, 0x81, 0x4a, 0x08, 0xf5, 0x42, 0x0f, 0x86, 0xf6, 0xd0, 0x9b, + 0x49, 0x8c, 0xb0, 0x48, 0x6c, 0x94, 0x18, 0x15, 0xfa, 0x27, 0xfb, 0x97, 0x2a, 0x27, 0x84, 0x72, + 0xe8, 0xc5, 0xf2, 0xe7, 0xf7, 0x9e, 0x35, 0xf3, 0x60, 0x74, 0xd0, 0xea, 0xb8, 0xa9, 0x27, 0xfb, + 0xca, 0x58, 0x83, 0x83, 0x8e, 0xd6, 0xe9, 0xa7, 0x0f, 0x64, 0x26, 0xac, 0xc0, 0x7b, 0x20, 0xab, + 0xd3, 0x5e, 0x52, 0x2f, 0xf1, 0xd9, 0xef, 0x29, 0x9d, 0x5c, 0x2c, 0x13, 0x27, 0x37, 0x87, 0xd3, + 0x79, 0xe3, 0x42, 0x6c, 0x53, 0xd4, 0x4f, 0x3c, 0x36, 0xe2, 0xed, 0x0f, 0xff, 0x21, 0xda, 0xa8, + 0x42, 0xd6, 0xea, 0x43, 0xd2, 0x20, 0xf1, 0x18, 0xe1, 0x17, 0xc6, 0x1b, 0x80, 0x75, 0x61, 0xb2, + 0x9d, 0x83, 0x9a, 0x92, 0x24, 0x60, 0x84, 0x5f, 0xbd, 0xb8, 0xec, 0x56, 0xd4, 0xdb, 0x66, 0x82, + 0x5e, 0x9b, 0xed, 0x18, 0xff, 0x41, 0x7f, 0x23, 0xb4, 0x39, 0x58, 0xda, 0x6f, 0x94, 0x33, 0xb9, + 0x19, 0x4a, 0x93, 0x4b, 0x1a, 0x26, 0x1e, 0x1b, 0xf3, 0xe6, 0x8e, 0x77, 0xd0, 0x2b, 0xad, 0x2a, + 0x25, 0x8d, 0x12, 0x8f, 0x0d, 0xa7, 0x7f, 0xae, 0xd6, 0x78, 0xd1, 0xea, 0xb8, 0x52, 0xa5, 0xe4, + 0xad, 0x23, 0x7d, 0x85, 0xa8, 0x5b, 0x0a, 0x43, 0x08, 0xb8, 0x78, 0x8f, 0x7f, 0xe1, 0x18, 0x06, + 0x33, 0x55, 0xc9, 0xcc, 0x9a, 0xea, 0x14, 0x7b, 0x18, 0x01, 0x99, 0xab, 0x42, 0xc6, 0x3e, 0x8e, + 0x20, 0x5a, 0x48, 0x2b, 0x72, 0x61, 0x45, 0x1c, 0xe0, 0x10, 0xc2, 0xe5, 0xa9, 0x2c, 0x94, 0xde, + 0xc5, 0xc4, 0x65, 0x9e, 0x1e, 0x17, 0xab, 0xe5, 0x56, 0x54, 0x79, 0xdc, 0x4b, 0x6f, 0xbf, 0x9d, + 0x6e, 0xad, 0x85, 0x2a, 0xe5, 0xb9, 0x58, 0x8f, 0x0d, 0xf8, 0x85, 0xd3, 0x37, 0x88, 0xba, 0x91, + 0x90, 0x42, 0xb8, 0x94, 0x99, 0xd1, 0x79, 0xdd, 0xf4, 0x1f, 0xf0, 0x0e, 0xf1, 0x01, 0xfe, 0xce, + 0x2b, 0x91, 0x59, 0x65, 0xb4, 0x28, 0x9e, 0x85, 0x36, 0xf5, 0xd9, 0xe7, 0x9a, 0x0f, 0xf9, 0xcf, + 0xe2, 0x57, 0x00, 0x00, 0x00, 0xff, 0xff, 0x1f, 0xc3, 0xf5, 0x68, 0xef, 0x01, 0x00, 0x00, } diff --git a/pb/unixfs.proto b/pb/unixfs.proto index ffc059e8b..5283dd7bd 100644 --- a/pb/unixfs.proto +++ b/pb/unixfs.proto @@ -19,8 +19,17 @@ message Data { optional uint64 hashType = 5; optional uint64 fanout = 6; + + // unixfs v1.5 metadata additions + optional uint32 mode = 7; + optional UnixTime mtime = 8; } message Metadata { optional string MimeType = 1; } + +message UnixTime { + required int64 Seconds = 1; + optional fixed32 FractionalNanoseconds = 2; +} diff --git a/unixfs.go b/unixfs.go index 026b8bb3f..337e0d89d 100644 --- a/unixfs.go +++ b/unixfs.go @@ -6,6 +6,8 @@ package unixfs import ( "errors" "fmt" + "os" + "time" proto "github.com/gogo/protobuf/proto" dag "github.com/ipfs/go-merkledag" @@ -69,7 +71,7 @@ func FilePBData(data []byte, totalsize uint64) []byte { return data } -//FolderPBData returns Bytes that represent a Directory. +// FolderPBData returns Bytes that represent a Directory. func FolderPBData() []byte { pbfile := new(pb.Data) typ := pb.Data_Directory @@ -77,13 +79,13 @@ func FolderPBData() []byte { data, err := proto.Marshal(pbfile) if err != nil { - //this really shouldnt happen, i promise + // this really shouldnt happen, i promise panic(err) } return data } -//WrapData marshals raw bytes into a `Data_Raw` type protobuf message. +// WrapData marshals raw bytes into a `Data_Raw` type protobuf message. func WrapData(b []byte) []byte { pbdata := new(pb.Data) typ := pb.Data_Raw @@ -100,7 +102,7 @@ func WrapData(b []byte) []byte { return out } -//SymlinkData returns a `Data_Symlink` protobuf message for the path you specify. +// SymlinkData returns a `Data_Symlink` protobuf message for the path you specify. func SymlinkData(path string) ([]byte, error) { pbdata := new(pb.Data) typ := pb.Data_Symlink @@ -304,6 +306,79 @@ func (n *FSNode) IsDir() bool { } } +// FileMode returns the optional file mode bits +func (n *FSNode) FileMode() os.FileMode { + // mask unknown bits as per https://github.com/ipfs/specs/blob/master/UNIXFS.md#metadata + mode := os.FileMode(n.format.GetMode() & 0o7777) + if mode != 0 { + return mode + } + + // Defaults if unset + switch n.Type() { + case pb.Data_Directory, pb.Data_HAMTShard: + return 0o0755 + case pb.Data_Raw, pb.Data_File, pb.Data_Symlink: + return 0o0644 + default: + return 0 + } +} + +// SetFileMode sets the file mode bits +func (n *FSNode) SetFileMode(m os.FileMode) { + // Special case, don't add a zero mode unless mode bits already exist + // This preserves binary compatibility with unixfs 1.0 + if m == 0 && n.format.Mode == nil { + return + } + + // mask unknown bits as per https://github.com/ipfs/specs/blob/master/UNIXFS.md#metadata + mode := (uint32(m) & 0o7777) | (n.format.GetMode() & 0xFFFFF000) + n.format.Mode = &mode +} + +// ModTime returns the optional modification time of the file. +func (n *FSNode) ModTime() time.Time { + mtime := n.format.GetMtime() + if mtime == nil { + // return an unspecified time + return time.Time{} + } + + secs := mtime.GetSeconds() + nsecs := uint32(0) + if mtime.FractionalNanoseconds != nil { + if *mtime.FractionalNanoseconds < 1 || *mtime.FractionalNanoseconds > 999999999 { + // Invalid time, return an unspecified time + return time.Time{} + } + nsecs = *mtime.FractionalNanoseconds + } + + return time.Unix(secs, int64(nsecs)) +} + +// SetModTime sets the modification time of the file. +func (n *FSNode) SetModTime(t time.Time) { + if t.IsZero() { + // unset the mtime since unix timestamp is not defined for an unitialized time value + n.format.Mtime = nil + return + } + unixnano := t.UnixNano() + + secs := unixnano / int64(time.Second) + nanos := uint32(unixnano - secs*int64(time.Second)) + + if n.format.Mtime == nil { + n.format.Mtime = &pb.UnixTime{} + } + + n.format.Mtime.Seconds = &secs + n.format.Mtime.FractionalNanoseconds = &nanos +} + // Metadata is used to store additional FSNode information. type Metadata struct { MimeType string diff --git a/unixfs_test.go b/unixfs_test.go index cf9e8548b..967972aa1 100644 --- a/unixfs_test.go +++ b/unixfs_test.go @@ -2,7 +2,9 @@ package unixfs import ( "bytes" + "os" "testing" + "time" proto "github.com/gogo/protobuf/proto" @@ -165,7 +167,6 @@ func TestMetadata(t *testing.T) { if !mimeAiff { t.Fatal("Metadata does not Marshal and Unmarshal properly!") } - } func TestIsDir(t *testing.T) { @@ -184,3 +185,108 @@ func TestIsDir(t *testing.T) { } } } + +func TestMtime(t *testing.T) { + fsn := NewFSNode(TFile) + fsn.SetData(make([]byte, 128)) + fsn.SetModTime(time.Unix(1638111600, 76552)) + + b, err := fsn.GetBytes() + if err != nil { + t.Fatal(err) + } + + pbn := new(pb.Data) + err = proto.Unmarshal(b, pbn) + if err != nil { + t.Fatal(err) + } + + if pbn.Mtime == nil { + t.Fatal("mtime is nil") + } + + if pbn.Mtime.Seconds == nil { + t.Fatal("mtime.Seconds is nil") + } + + if *pbn.Mtime.Seconds != 1638111600 { + t.Errorf("got mtime seconds %d, wanted %d", *pbn.Mtime.Seconds, 1638111600) + } + + if pbn.Mtime.FractionalNanoseconds == nil { + t.Fatal("mtime.FractionalNanoseconds is nil") + } + + if *pbn.Mtime.FractionalNanoseconds != 76552 { + t.Errorf("got mtime seconds %d, wanted %d", *pbn.Mtime.FractionalNanoseconds, 76552) + } +} + +func TestMode(t *testing.T) { + fsn := NewFSNode(TFile) + fsn.SetData(make([]byte, 128)) + fsn.SetFileMode(os.FileMode(0o521)) + + b, err := fsn.GetBytes() + if err != nil { + t.Fatal(err) + } + + pbn := new(pb.Data) + err = proto.Unmarshal(b, pbn) + if err != nil { + t.Fatal(err) + } + + if pbn.Mode == nil { + t.Fatal("mode is nil") + } + + if *pbn.Mode != 0o521 { + t.Errorf("got mode %d, wanted %d", *pbn.Mode, 0o521) + } +} + +func TestModePreservesUnknownBits(t *testing.T) { + pbn := new(pb.Data) + typ := pb.Data_File + pbn.Type = &typ + + // set mode with undefined bits set + origmode := uint32(0o12340755) + pbn.Mode = &origmode + + b, err := proto.Marshal(pbn) + if err != nil { + t.Fatalf("failed to marshal protobuf: %v", err) + } + + fsn, err := FSNodeFromBytes(b) + if err != nil { + t.Fatalf("failed to unmarshal protobuf: %v", err) + } + + // set mode should not affect undefined bits + fsn.SetFileMode(os.FileMode(0o0521)) + + // Marshal + b2, err := proto.Marshal(&fsn.format) + if err != nil { + t.Fatalf("failed to marshal protobuf: %v", err) + } + + pbn2 := new(pb.Data) + err = proto.Unmarshal(b2, pbn2) + if err != nil { + t.Fatal(err) + } + + if pbn2.Mode == nil { + t.Fatal("mode is nil") + } + + if *pbn2.Mode != 0o12340521 { + t.Errorf("got mode %o, wanted %o", *pbn2.Mode, 0o12340521) + } +} From f455d36c5fd1bab0395f3c441f80cbd72454a180 Mon Sep 17 00:00:00 2001 From: Ian Davis Date: Thu, 13 Jan 2022 12:48:13 +0000 Subject: [PATCH 2/2] Use nil for mtime.FractionalNanoseconds instead of zero --- unixfs.go | 11 ++++++++++- unixfs_test.go | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/unixfs.go b/unixfs.go index 337e0d89d..bf70d228c 100644 --- a/unixfs.go +++ b/unixfs.go @@ -376,7 +376,16 @@ func (n *FSNode) SetModTime(t time.Time) { } n.format.Mtime.Seconds = &secs - n.format.Mtime.FractionalNanoseconds = &nanos + + // From spec https://github.com/ipfs/specs/blob/master/UNIXFS.md#metadata + // An mtime structure with FractionalNanoseconds outside of the on-wire range [1, 999999999] is not valid. This includes + // a fractional value of 0. Implementations encountering such values should consider the entire enclosing metadata + // block malformed and abort processing the corresponding DAG. + if nanos == 0 { + n.format.Mtime.FractionalNanoseconds = nil + } else { + n.format.Mtime.FractionalNanoseconds = &nanos + } } // Metadata is used to store additional FSNode information. diff --git a/unixfs_test.go b/unixfs_test.go index 967972aa1..35716e601 100644 --- a/unixfs_test.go +++ b/unixfs_test.go @@ -223,6 +223,39 @@ func TestMtime(t *testing.T) { } } +func TestMtimeWholeSeconds(t *testing.T) { + fsn := NewFSNode(TFile) + fsn.SetData(make([]byte, 128)) + fsn.SetModTime(time.Unix(1638111600, 0)) // filesystems such as NFS only have 1sec resolution for mtime + + b, err := fsn.GetBytes() + if err != nil { + t.Fatal(err) + } + + pbn := new(pb.Data) + err = proto.Unmarshal(b, pbn) + if err != nil { + t.Fatal(err) + } + + if pbn.Mtime == nil { + t.Fatal("mtime is nil") + } + + if pbn.Mtime.Seconds == nil { + t.Fatal("mtime.Seconds is nil") + } + + if *pbn.Mtime.Seconds != 1638111600 { + t.Errorf("got mtime seconds %d, wanted %d", *pbn.Mtime.Seconds, 1638111600) + } + + if pbn.Mtime.FractionalNanoseconds != nil { + t.Fatalf("got mtime.FractionalNanoseconds %v, wanted nil", *pbn.Mtime.FractionalNanoseconds) + } +} + func TestMode(t *testing.T) { fsn := NewFSNode(TFile) fsn.SetData(make([]byte, 128))