Migrate go-unixfs into unixfs/ipld-merkledag

go.mod

@@ -3,51 +3,84 @@ module github.com/ipfs/go-libipfs
 go 1.19
 
 require (
+	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137
 	github.com/benbjohnson/clock v1.3.0
 	github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3
+	github.com/gogo/protobuf v1.3.2
 	github.com/gorilla/mux v1.8.0
+	github.com/ipfs/go-bitfield v1.0.0
+	github.com/ipfs/go-block-format v0.0.3
+	github.com/ipfs/go-blockservice v0.5.0
 	github.com/ipfs/go-cid v0.3.2
+	github.com/ipfs/go-datastore v0.6.0
+	github.com/ipfs/go-ipfs-blockstore v1.2.0
+	github.com/ipfs/go-ipfs-chunker v0.0.5
+	github.com/ipfs/go-ipfs-exchange-offline v0.3.0
+	github.com/ipfs/go-ipfs-posinfo v0.0.1
+	github.com/ipfs/go-ipfs-util v0.0.2
+	github.com/ipfs/go-ipld-format v0.3.1
 	github.com/ipfs/go-ipns v0.3.0
+	github.com/ipfs/go-log v1.0.5
 	github.com/ipfs/go-log/v2 v2.5.1
+	github.com/ipfs/go-merkledag v0.9.0
 	github.com/libp2p/go-libp2p v0.23.4
 	github.com/libp2p/go-libp2p-record v0.2.0
 	github.com/multiformats/go-multiaddr v0.8.0
 	github.com/multiformats/go-multibase v0.1.1
 	github.com/multiformats/go-multihash v0.2.1
 	github.com/samber/lo v1.36.0
+	github.com/spaolacci/murmur3 v1.1.0
 	github.com/stretchr/testify v1.8.1
 	go.opencensus.io v0.23.0
+	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
 	golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab
 )
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
-	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/ipfs/go-ipfs-util v0.0.2 // indirect
-	github.com/ipld/go-ipld-prime v0.9.0 // indirect
-	github.com/klauspost/cpuid/v2 v2.1.1 // indirect
+	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/hashicorp/golang-lru v0.5.4 // indirect
+	github.com/ipfs/bbloom v0.0.4 // indirect
+	github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
+	github.com/ipfs/go-ipfs-exchange-interface v0.2.0 // indirect
+	github.com/ipfs/go-ipld-cbor v0.0.6 // indirect
+	github.com/ipfs/go-ipld-legacy v0.1.1 // indirect
+	github.com/ipfs/go-metrics-interface v0.0.1 // indirect
+	github.com/ipfs/go-verifcid v0.0.2 // indirect
+	github.com/ipld/go-codec-dagpb v1.5.0 // indirect
+	github.com/ipld/go-ipld-prime v0.18.0 // indirect
+	github.com/jbenet/goprocess v0.1.4 // indirect
+	github.com/klauspost/cpuid/v2 v2.1.2 // indirect
 	github.com/libp2p/go-buffer-pool v0.1.0 // indirect
 	github.com/libp2p/go-openssl v0.1.0 // indirect
-	github.com/mattn/go-isatty v0.0.16 // indirect
+	github.com/mattn/go-isatty v0.0.17 // indirect
 	github.com/mattn/go-pointer v0.0.1 // indirect
 	github.com/minio/sha256-simd v1.0.0 // indirect
 	github.com/mr-tron/base58 v1.2.0 // indirect
 	github.com/multiformats/go-base32 v0.1.0 // indirect
 	github.com/multiformats/go-base36 v0.1.0 // indirect
 	github.com/multiformats/go-multicodec v0.6.0 // indirect
-	github.com/multiformats/go-varint v0.0.6 // indirect
+	github.com/multiformats/go-varint v0.0.7 // indirect
+	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1 // indirect
+	github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e // indirect
 	github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect
-	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/stretchr/objx v0.5.0 // indirect
+	github.com/whyrusleeping/cbor-gen v0.0.0-20221220214510-0333c149dec0 // indirect
+	github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
+	go.opentelemetry.io/otel v1.7.0 // indirect
+	go.opentelemetry.io/otel/trace v1.7.0 // indirect
 	go.uber.org/atomic v1.10.0 // indirect
 	go.uber.org/multierr v1.8.0 // indirect
 	go.uber.org/zap v1.23.0 // indirect
 	golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
 	golang.org/x/exp v0.0.0-20220916125017-b168a2c6b86b // indirect
+	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	lukechampine.com/blake3 v1.1.7 // indirect
 )

unixfs/ipld-merkledag/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014-2018 Juan Batiz-Benet
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

unixfs/ipld-merkledag/README.md

@@ -0,0 +1,66 @@
+go-unixfs
+==================
+
+[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io)
+[![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/)
+[![](https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs)
+[![Coverage Status](https://codecov.io/gh/ipfs/go-unixfs/branch/master/graph/badge.svg)](https://codecov.io/gh/ipfs/go-unixfs/branch/master)
+[![Travis CI](https://travis-ci.org/ipfs/go-unixfs.svg?branch=master)](https://travis-ci.org/ipfs/go-unixfs)
+
+> go-unixfs implements unix-like filesystem utilities on top of an ipld merkledag
+
+## Lead Maintainer
+
+[Steven Allen](https://github.com/Stebalien)
+
+## Table of Contents
+
+- [Directory](#directory)
+- [Install](#install)
+- [Contribute](#contribute)
+- [License](#license)
+
+## Package Directory
+This package contains many subpackages, each of which can be very large on its own.
+
+### Top Level
+The top level unixfs package defines the unixfs format datastructures, and some helper methods around it.
+
+### importers
+The `importer` subpackage is what you'll use when you want to turn a normal file into a unixfs file.
+
+### io
+The `io` subpackage provides helpers for reading files and manipulating directories. The `DagReader` takes a
+reference to a unixfs file and returns a file handle that can be read from and seeked through. The `Directory`
+interface allows you to easily read items in a directory, add items to a directory, and do lookups.
+
+### mod
+The `mod` subpackage implements a `DagModifier` type that can be used to write to an existing unixfs file, or
+create a new one. The logic for this is significantly more complicated than for the dagreader, so its a separate
+type. (TODO: maybe it still belongs in the `io` subpackage though?)
+
+### hamt
+The `hamt` subpackage implements a CHAMP hamt that is used in unixfs directory sharding.
+
+### archive
+The `archive` subpackage implements a `tar` importer and exporter. The objects created here are not officially unixfs,
+but in the future, this may be integrated more directly.
+
+### test
+The `test` subpackage provides several utilities to make testing unixfs related things easier.
+
+## Install
+
+```sh
+go get github.com/ipfs/go-unixfs
+```
+
+## Contribute
+
+PRs are welcome!
+
+Small note: If editing the Readme, please conform to the [standard-readme](https://github.com/RichardLitt/standard-readme) specification.
+
+## License
+
+MIT © Juan Batiz-Benet

unixfs/ipld-merkledag/file/unixfile.go

@@ -0,0 +1,183 @@
+package unixfile
+
+import (
+	"context"
+	"errors"
+
+	ft "github.com/ipfs/go-libipfs/unixfs/ipld-merkledag"
+	uio "github.com/ipfs/go-libipfs/unixfs/ipld-merkledag/io"
+
+	ipld "github.com/ipfs/go-ipld-format"
+	"github.com/ipfs/go-libipfs/files"
+	dag "github.com/ipfs/go-merkledag"
+)
+
+// Number to file to prefetch in directories
+// TODO: should we allow setting this via context hint?
+const prefetchFiles = 4
+
+type ufsDirectory struct {
+	ctx   context.Context
+	dserv ipld.DAGService
+	dir   uio.Directory
+	size  int64
+}
+
+type ufsIterator struct {
+	ctx   context.Context
+	files chan *ipld.Link
+	dserv ipld.DAGService
+
+	curName string
+	curFile files.Node
+
+	err   error
+	errCh chan error
+}
+
+func (it *ufsIterator) Name() string {
+	return it.curName
+}
+
+func (it *ufsIterator) Node() files.Node {
+	return it.curFile
+}
+
+func (it *ufsIterator) Next() bool {
+	if it.err != nil {
+		return false
+	}
+
+	var l *ipld.Link
+	var ok bool
+	for !ok {
+		if it.files == nil && it.errCh == nil {
+			return false
+		}
+		select {
+		case l, ok = <-it.files:
+			if !ok {
+				it.files = nil
+			}
+		case err := <-it.errCh:
+			it.errCh = nil
+			it.err = err
+
+			if err != nil {
+				return false
+			}
+		}
+	}
+
+	it.curFile = nil
+
+	nd, err := l.GetNode(it.ctx, it.dserv)
+	if err != nil {
+		it.err = err
+		return false
+	}
+
+	it.curName = l.Name
+	it.curFile, it.err = NewUnixfsFile(it.ctx, it.dserv, nd)
+	return it.err == nil
+}
+
+func (it *ufsIterator) Err() error {
+	return it.err
+}
+
+func (d *ufsDirectory) Close() error {
+	return nil
+}
+
+func (d *ufsDirectory) Entries() files.DirIterator {
+	fileCh := make(chan *ipld.Link, prefetchFiles)
+	errCh := make(chan error, 1)
+	go func() {
+		errCh <- d.dir.ForEachLink(d.ctx, func(link *ipld.Link) error {
+			if d.ctx.Err() != nil {
+				return d.ctx.Err()
+			}
+			select {
+			case fileCh <- link:
+			case <-d.ctx.Done():
+				return d.ctx.Err()
+			}
+			return nil
+		})
+
+		close(errCh)
+		close(fileCh)
+	}()
+
+	return &ufsIterator{
+		ctx:   d.ctx,
+		files: fileCh,
+		errCh: errCh,
+		dserv: d.dserv,
+	}
+}
+
+func (d *ufsDirectory) Size() (int64, error) {
+	return d.size, nil
+}
+
+type ufsFile struct {
+	uio.DagReader
+}
+
+func (f *ufsFile) Size() (int64, error) {
+	return int64(f.DagReader.Size()), nil
+}
+
+func newUnixfsDir(ctx context.Context, dserv ipld.DAGService, nd *dag.ProtoNode) (files.Directory, error) {
+	dir, err := uio.NewDirectoryFromNode(dserv, nd)
+	if err != nil {
+		return nil, err
+	}
+
+	size, err := nd.Size()
+	if err != nil {
+		return nil, err
+	}
+
+	return &ufsDirectory{
+		ctx:   ctx,
+		dserv: dserv,
+
+		dir:  dir,
+		size: int64(size),
+	}, nil
+}
+
+func NewUnixfsFile(ctx context.Context, dserv ipld.DAGService, nd ipld.Node) (files.Node, error) {
+	switch dn := nd.(type) {
+	case *dag.ProtoNode:
+		fsn, err := ft.FSNodeFromBytes(dn.Data())
+		if err != nil {
+			return nil, err
+		}
+		if fsn.IsDir() {
+			return newUnixfsDir(ctx, dserv, dn)
+		}
+		if fsn.Type() == ft.TSymlink {
+			return files.NewLinkFile(string(fsn.Data()), nil), nil
+		}
+
+	case *dag.RawNode:
+	default:
+		return nil, errors.New("unknown node type")
+	}
+
+	dr, err := uio.NewDagReader(ctx, nd, dserv)
+	if err != nil {
+		return nil, err
+	}
+
+	return &ufsFile{
+		DagReader: dr,
+	}, nil
+}
+
+var _ files.Directory = &ufsDirectory{}
+var _ files.File = &ufsFile{}