Fix Tsize values, fix HAMT and make recursive builder output match go-unixfs

data/builder/dir_test.go

@@ -3,12 +3,16 @@ package builder
 import (
 	"bytes"
 	"fmt"
+	"os"
+	"path/filepath"
 	"testing"
 
+	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-unixfsnode"
 	dagpb "github.com/ipld/go-codec-dagpb"
 	"github.com/ipld/go-ipld-prime"
 	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
+	"github.com/stretchr/testify/require"
 )
 
 func mkEntries(cnt int, ls *ipld.LinkSystem) ([]dagpb.PBLink, error) {
@@ -41,7 +45,7 @@ func TestBuildUnixFSDirectory(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		dl, err := BuildUnixFSDirectory(entries, &ls)
+		dl, _, err := BuildUnixFSDirectory(entries, &ls)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -70,3 +74,124 @@ func TestBuildUnixFSDirectory(t *testing.T) {
 		}
 	}
 }
+
+func TestBuildUnixFSRecursive(t *testing.T) {
+	// only the top CID is of interest, but this tree is correct and can be used for future validation
+	fixture := fentry{
+		"rootDir",
+		"",
+		mustCidDecode("bafybeihswl3f7pa7fueyayewcvr3clkdz7oetv4jolyejgw26p6l3qzlbm"),
+		[]fentry{
+			{"a", "aaa", mustCidDecode("bafkreieygsdw3t5qlsywpjocjfj6xjmmjlejwgw7k7zi6l45bgxra7xi6a"), nil},
+			{
+				"b",
+				"",
+				mustCidDecode("bafybeibohj54uixf2mso4t53suyarv6cfuxt6b5cj6qjsqaa2ezfxnu5pu"),
+				[]fentry{
+					{"1", "111", mustCidDecode("bafkreihw4cq6flcbsrnjvj77rkfkudhlyevdxteydkjjvvopqefasdqrvy"), nil},
+					{"2", "222", mustCidDecode("bafkreie3q4kremt4bhhjdxletm7znjr3oqeo6jt4rtcxcaiu4yuxgdfwd4"), nil},
+				},
+			},
+			{"c", "ccc", mustCidDecode("bafkreide3ksevvet74uks3x7vnxhp4ltfi6zpwbsifmbwn6324fhusia7y"), nil},
+		},
+	}
+
+	ls := cidlink.DefaultLinkSystem()
+	storage := cidlink.Memory{}
+	ls.StorageReadOpener = storage.OpenRead
+	ls.StorageWriteOpener = storage.OpenWrite
+
+	dir := t.TempDir()
+	makeFixture(t, dir, fixture)
+
+	lnk, sz, err := BuildUnixFSRecursive(filepath.Join(dir, fixture.name), &ls)
+	require.NoError(t, err)
+	require.Equal(t, fixture.expectedLnk.String(), lnk.String())
+	require.Equal(t, uint64(245), sz)
+}
+
+func TestBuildUnixFSRecursiveLargeSharded(t *testing.T) {
+	// only the top CID is of interest, but this tree is correct and can be used for future validation
+	fixture := fentry{
+		"rootDir",
+		"",
+		mustCidDecode("bafybeigyvxs6og5jbmpaa43qbhhd5swklqcfzqdrtjgfh53qjon6hpjaye"),
+		make([]fentry, 0),
+	}
+
+	for i := 0; i < 1344; i++ {
+		name := fmt.Sprintf("long name to fill out bytes to make the sharded directory test flip over the sharded directory limit because link names are included in the directory entry %d", i)
+		fixture.children = append(fixture.children, fentry{name, name, cid.Undef, nil})
+	}
+
+	ls := cidlink.DefaultLinkSystem()
+	storage := cidlink.Memory{}
+	ls.StorageReadOpener = storage.OpenRead
+	ls.StorageWriteOpener = storage.OpenWrite
+
+	dir := t.TempDir()
+	makeFixture(t, dir, fixture)
+
+	lnk, sz, err := BuildUnixFSRecursive(filepath.Join(dir, fixture.name), &ls)
+	require.NoError(t, err)
+	require.Equal(t, fixture.expectedLnk.String(), lnk.String())
+	require.Equal(t, uint64(515735), sz)
+}
+
+// Same as TestBuildUnixFSRecursiveLargeSharded but it's one file less which flips
+// it back to the un-sharded format. So we're testing the boundary condition and
+// the proper construction of large DAGs.
+func TestBuildUnixFSRecursiveLargeUnsharded(t *testing.T) {
+	// only the top CID is of interest, but this tree is correct and can be used for future validation
+	fixture := fentry{
+		"rootDir",
+		"",
+		mustCidDecode("bafybeihecq4rpl4nw3cgfb2uiwltgsmw5sutouvuldv5fxn4gfbihvnalq"),
+		make([]fentry, 0),
+	}
+
+	for i := 0; i < 1343; i++ {
+		name := fmt.Sprintf("long name to fill out bytes to make the sharded directory test flip over the sharded directory limit because link names are included in the directory entry %d", i)
+		fixture.children = append(fixture.children, fentry{name, name, cid.Undef, nil})
+	}
+
+	ls := cidlink.DefaultLinkSystem()
+	storage := cidlink.Memory{}
+	ls.StorageReadOpener = storage.OpenRead
+	ls.StorageWriteOpener = storage.OpenWrite
+
+	dir := t.TempDir()
+	makeFixture(t, dir, fixture)
+
+	lnk, sz, err := BuildUnixFSRecursive(filepath.Join(dir, fixture.name), &ls)
+	require.NoError(t, err)
+	require.Equal(t, fixture.expectedLnk.String(), lnk.String())
+	require.Equal(t, uint64(490665), sz)
+}
+
+type fentry struct {
+	name        string
+	content     string
+	expectedLnk cid.Cid
+	children    []fentry
+}
+
+func makeFixture(t *testing.T, dir string, fixture fentry) {
+	path := filepath.Join(dir, fixture.name)
+	if fixture.children != nil {
+		require.NoError(t, os.Mkdir(path, 0755))
+		for _, c := range fixture.children {
+			makeFixture(t, path, c)
+		}
+	} else {
+		os.WriteFile(path, []byte(fixture.content), 0644)
+	}
+}
+
+func mustCidDecode(s string) cid.Cid {
+	c, err := cid.Decode(s)
+	if err != nil {
+		panic(err)
+	}
+	return c
+}

data/builder/directory.go

@@ -30,6 +30,7 @@ func BuildUnixFSRecursive(root string, ls *ipld.LinkSystem) (ipld.Link, uint64,
 	m := info.Mode()
 	switch {
 	case m.IsDir():
+		var tsize uint64
 		entries, err := os.ReadDir(root)
 		if err != nil {
 			return nil, 0, err
@@ -40,27 +41,35 @@ func BuildUnixFSRecursive(root string, ls *ipld.LinkSystem) (ipld.Link, uint64,
 			if err != nil {
 				return nil, 0, err
 			}
+			tsize += sz
 			entry, err := BuildUnixFSDirectoryEntry(e.Name(), int64(sz), lnk)
 			if err != nil {
 				return nil, 0, err
 			}
 			lnks = append(lnks, entry)
 		}
-		outLnk, err := BuildUnixFSDirectory(lnks, ls)
-		return outLnk, 0, err
+		return BuildUnixFSDirectory(lnks, ls)
 	case m.Type() == fs.ModeSymlink:
 		content, err := os.Readlink(root)
 		if err != nil {
 			return nil, 0, err
 		}
-		return BuildUnixFSSymlink(content, ls)
+		outLnk, sz, err := BuildUnixFSSymlink(content, ls)
+		if err != nil {
+			return nil, 0, err
+		}
+		return outLnk, sz, nil
 	case m.IsRegular():
 		fp, err := os.Open(root)
 		if err != nil {
 			return nil, 0, err
 		}
 		defer fp.Close()
-		return BuildUnixFSFile(fp, "", ls)
+		outLnk, sz, err := BuildUnixFSFile(fp, "", ls)
+		if err != nil {
+			return nil, 0, err
+		}
+		return outLnk, sz, nil
 	default:
 		return nil, 0, fmt.Errorf("cannot encode non regular file: %s", root)
 	}
@@ -87,46 +96,52 @@ func estimateDirSize(entries []dagpb.PBLink) int {
 }
 
 // BuildUnixFSDirectory creates a directory link over a collection of entries.
-func BuildUnixFSDirectory(entries []dagpb.PBLink, ls *ipld.LinkSystem) (ipld.Link, error) {
+func BuildUnixFSDirectory(entries []dagpb.PBLink, ls *ipld.LinkSystem) (ipld.Link, uint64, error) {
 	if estimateDirSize(entries) > shardSplitThreshold {
 		return BuildUnixFSShardedDirectory(defaultShardWidth, multihash.MURMUR3X64_64, entries, ls)
 	}
 	ufd, err := BuildUnixFS(func(b *Builder) {
 		DataType(b, data.Data_Directory)
 	})
 	if err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 	pbb := dagpb.Type.PBNode.NewBuilder()
 	pbm, err := pbb.BeginMap(2)
 	if err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 	if err = pbm.AssembleKey().AssignString("Data"); err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 	if err = pbm.AssembleValue().AssignBytes(data.EncodeUnixFSData(ufd)); err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 	if err = pbm.AssembleKey().AssignString("Links"); err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 	lnks, err := pbm.AssembleValue().BeginList(int64(len(entries)))
 	if err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 	// sorting happens in codec-dagpb
+	var totalSize uint64
 	for _, e := range entries {
+		totalSize += uint64(e.Tsize.Must().Int())
 		if err := lnks.AssembleValue().AssignNode(e); err != nil {
-			return nil, err
+			return nil, 0, err
 		}
 	}
 	if err := lnks.Finish(); err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 	if err := pbm.Finish(); err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 	node := pbb.Build()
-	return ls.Store(ipld.LinkContext{}, fileLinkProto, node)
+	lnk, sz, err := sizedStore(ls, fileLinkProto, node)
+	if err != nil {
+		return nil, 0, err
+	}
+	return lnk, totalSize + sz, err
 }