unixfs/ipld-merkledag: merge main

Resolve conflicts

.github/workflows/test-examples.yml

@@ -0,0 +1,54 @@
+on: [push, pull_request]
+name: Go Test Examples
+
+jobs:
+  unit:
+    defaults:
+      run:
+        working-directory: examples
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ "ubuntu", "windows", "macos" ]
+        go: [ "1.18.x", "1.19.x" ]
+    env:
+      COVERAGES: ""
+    runs-on: ${{ format('{0}-latest', matrix.os) }}
+    name: ${{ matrix.os }} (go ${{ matrix.go }})
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: recursive
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ${{ matrix.go }}
+      - name: Go information
+        run: |
+          go version
+          go env
+      - name: Use msys2 on windows
+        if: ${{ matrix.os == 'windows' }}
+        shell: bash
+        # The executable for msys2 is also called bash.cmd
+        #   https://github.com/actions/virtual-environments/blob/main/images/win/Windows2019-Readme.md#shells
+        # If we prepend its location to the PATH
+        #   subsequent 'shell: bash' steps will use msys2 instead of gitbash
+        run: echo "C:/msys64/usr/bin" >> $GITHUB_PATH
+      - name: Run tests
+        uses: protocol/multiple-go-modules@v1.2
+        with:
+          run: go test -v -shuffle=on ./...
+      - name: Run tests (32 bit)
+        if: ${{ matrix.os != 'macos' }} # can't run 32 bit tests on OSX.
+        uses: protocol/multiple-go-modules@v1.2
+        env:
+          GOARCH: 386
+        with:
+          run: |
+            export "PATH=${{ env.PATH_386 }}:$PATH"
+            go test -v -shuffle=on ./...
+      - name: Run tests with race detector
+        if: ${{ matrix.os == 'ubuntu' }} # speed things up. Windows and OSX VMs are slow
+        uses: protocol/multiple-go-modules@v1.2
+        with:
+          run: go test -v -race ./...

examples/README.md

@@ -0,0 +1,9 @@
+# go-libipfs examples and tutorials
+
+In this folder, you can find some examples to help you get started using go-libipfs and its associated libraries in your applications.
+
+Let us know if you find any issue or if you want to contribute and add a new tutorial, feel welcome to submit a pr, thank you!
+
+## Examples and Tutorials
+
+- [Gateway backed by a CAR file](./gateway-car)

examples/gateway-car/README.md

@@ -0,0 +1,32 @@
+# Gateway backed by a CAR File
+
+This is an example that shows how to build a Gateway backed by the contents of
+a CAR file. A [CAR file](https://ipld.io/specs/transport/car/) is a Content
+Addressable aRchive that contains blocks.
+
+## Build
+
+```bash
+> go build -o gateway
+```
+
+## Usage
+
+First of all, you will need some content stored as a CAR file. You can easily
+export your favorite website, or content, using:
+
+```
+ipfs dag export <CID> > data.car
+```
+
+Then, you can start the gateway with:
+
+
+```
+./gateway -c data.car -p 8040
+```
+
+Now you can access the gateway in [127.0.0.1:8040](http://127.0.0.1:8040). It will
+behave like a regular IPFS Gateway, except for the fact that all contents are provided
+from the CAR file. Therefore, things such as IPNS resolution and fetching contents
+from nodes in the IPFS network won't work.

examples/gateway-car/api.go

@@ -0,0 +1,174 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	gopath "path"
+
+	"github.com/ipfs/go-blockservice"
+	"github.com/ipfs/go-cid"
+	bsfetcher "github.com/ipfs/go-fetcher/impl/blockservice"
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
+	format "github.com/ipfs/go-ipld-format"
+	"github.com/ipfs/go-libipfs/blocks"
+	"github.com/ipfs/go-libipfs/files"
+	"github.com/ipfs/go-merkledag"
+	ipfspath "github.com/ipfs/go-path"
+	"github.com/ipfs/go-path/resolver"
+	"github.com/ipfs/go-unixfs"
+	ufile "github.com/ipfs/go-unixfs/file"
+	uio "github.com/ipfs/go-unixfs/io"
+	"github.com/ipfs/go-unixfsnode"
+	iface "github.com/ipfs/interface-go-ipfs-core"
+	ifacepath "github.com/ipfs/interface-go-ipfs-core/path"
+	dagpb "github.com/ipld/go-codec-dagpb"
+	"github.com/ipld/go-ipld-prime"
+	"github.com/ipld/go-ipld-prime/node/basicnode"
+	"github.com/ipld/go-ipld-prime/schema"
+)
+
+type blocksGateway struct {
+	blockStore   blockstore.Blockstore
+	blockService blockservice.BlockService
+	dagService   format.DAGService
+	resolver     resolver.Resolver
+}
+
+func newBlocksGateway(blockService blockservice.BlockService) (*blocksGateway, error) {
+	// Setup the DAG services, which use the CAR block store.
+	dagService := merkledag.NewDAGService(blockService)
+
+	// Setup the UnixFS resolver.
+	fetcherConfig := bsfetcher.NewFetcherConfig(blockService)
+	fetcherConfig.PrototypeChooser = dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) {
+		if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok {
+			return tlnkNd.LinkTargetNodePrototype(), nil
+		}
+		return basicnode.Prototype.Any, nil
+	})
+	fetcher := fetcherConfig.WithReifier(unixfsnode.Reify)
+	resolver := resolver.NewBasicResolver(fetcher)
+
+	return &blocksGateway{
+		blockStore:   blockService.Blockstore(),
+		blockService: blockService,
+		dagService:   dagService,
+		resolver:     resolver,
+	}, nil
+}
+
+func (api *blocksGateway) GetUnixFsNode(ctx context.Context, p ifacepath.Resolved) (files.Node, error) {
+	nd, err := api.resolveNode(ctx, p)
+	if err != nil {
+		return nil, err
+	}
+
+	return ufile.NewUnixfsFile(ctx, api.dagService, nd)
+}
+
+func (api *blocksGateway) LsUnixFsDir(ctx context.Context, p ifacepath.Resolved) (<-chan iface.DirEntry, error) {
+	node, err := api.resolveNode(ctx, p)
+	if err != nil {
+		return nil, err
+	}
+
+	dir, err := uio.NewDirectoryFromNode(api.dagService, node)
+	if err != nil {
+		return nil, err
+	}
+
+	out := make(chan iface.DirEntry, uio.DefaultShardWidth)
+
+	go func() {
+		defer close(out)
+		for l := range dir.EnumLinksAsync(ctx) {
+			select {
+			case out <- api.processLink(ctx, l):
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+
+	return out, nil
+}
+
+func (api *blocksGateway) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) {
+	return api.blockService.GetBlock(ctx, c)
+}
+
+func (api *blocksGateway) GetIPNSRecord(context.Context, cid.Cid) ([]byte, error) {
+	return nil, errors.New("not implemented")
+}
+
+func (api *blocksGateway) IsCached(ctx context.Context, p ifacepath.Path) bool {
+	rp, err := api.ResolvePath(ctx, p)
+	if err != nil {
+		return false
+	}
+
+	has, _ := api.blockStore.Has(ctx, rp.Cid())
+	return has
+}
+
+func (api *blocksGateway) ResolvePath(ctx context.Context, p ifacepath.Path) (ifacepath.Resolved, error) {
+	if _, ok := p.(ifacepath.Resolved); ok {
+		return p.(ifacepath.Resolved), nil
+	}
+
+	if err := p.IsValid(); err != nil {
+		return nil, err
+	}
+
+	if p.Namespace() != "ipfs" {
+		return nil, fmt.Errorf("unsupported path namespace: %s", p.Namespace())
+	}
+
+	ipath := ipfspath.Path(p.String())
+	node, rest, err := api.resolver.ResolveToLastNode(ctx, ipath)
+	if err != nil {
+		return nil, err
+	}
+
+	root, err := cid.Parse(ipath.Segments()[1])
+	if err != nil {
+		return nil, err
+	}
+
+	return ifacepath.NewResolvedPath(ipath, node, root, gopath.Join(rest...)), nil
+}
+
+func (api *blocksGateway) resolveNode(ctx context.Context, p ifacepath.Path) (format.Node, error) {
+	rp, err := api.ResolvePath(ctx, p)
+	if err != nil {
+		return nil, err
+	}
+
+	node, err := api.dagService.Get(ctx, rp.Cid())
+	if err != nil {
+		return nil, fmt.Errorf("get node: %w", err)
+	}
+	return node, nil
+}
+
+func (api *blocksGateway) processLink(ctx context.Context, result unixfs.LinkResult) iface.DirEntry {
+	if result.Err != nil {
+		return iface.DirEntry{Err: result.Err}
+	}
+
+	link := iface.DirEntry{
+		Name: result.Link.Name,
+		Cid:  result.Link.Cid,
+	}
+
+	switch link.Cid.Type() {
+	case cid.Raw:
+		link.Type = iface.TFile
+		link.Size = result.Link.Size
+	case cid.DagProtobuf:
+		link.Size = result.Link.Size
+	}
+
+	return link
+}

examples/gateway-car/main.go

@@ -0,0 +1,81 @@
+package main
+
+import (
+	"flag"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"strconv"
+
+	"github.com/ipfs/go-blockservice"
+	"github.com/ipfs/go-cid"
+	offline "github.com/ipfs/go-ipfs-exchange-offline"
+	"github.com/ipfs/go-libipfs/gateway"
+	carblockstore "github.com/ipld/go-car/v2/blockstore"
+)
+
+func main() {
+	carFilePtr := flag.String("c", "", "path to CAR file to back this gateway from")
+	portPtr := flag.Int("p", 8080, "port to run this gateway from")
+	flag.Parse()
+
+	blockService, roots, f, err := newBlockServiceFromCAR(*carFilePtr)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer f.Close()
+
+	gateway, err := newBlocksGateway(blockService)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	handler := newHandler(gateway, *portPtr)
+
+	address := "127.0.0.1:" + strconv.Itoa(*portPtr)
+	log.Printf("Listening on http://%s", address)
+	for _, cid := range roots {
+		log.Printf("Hosting CAR root at http://%s/ipfs/%s", address, cid.String())
+	}
+
+	if err := http.ListenAndServe(address, handler); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func newBlockServiceFromCAR(filepath string) (blockservice.BlockService, []cid.Cid, io.Closer, error) {
+	r, err := os.Open(filepath)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	bs, err := carblockstore.NewReadOnly(r, nil)
+	if err != nil {
+		_ = r.Close()
+		return nil, nil, nil, err
+	}
+
+	roots, err := bs.Roots()
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	blockService := blockservice.New(bs, offline.Exchange(bs))
+	return blockService, roots, r, nil
+}
+
+func newHandler(gw *blocksGateway, port int) http.Handler {
+	headers := map[string][]string{}
+	gateway.AddAccessControlHeaders(headers)
+
+	conf := gateway.Config{
+		Headers: headers,
+	}
+
+	mux := http.NewServeMux()
+	gwHandler := gateway.NewHandler(conf, gw)
+	mux.Handle("/ipfs/", gwHandler)
+	mux.Handle("/ipns/", gwHandler)
+	return mux
+}