Merge pull request ipfs/go-fetcher#13 from ipfs/feat/augment-node-bui…

…lder-chooser

Add NodeReifier to fetcher config

This commit was moved from ipfs/go-fetcher@3290471

fetcher/README.md

@@ -4,7 +4,7 @@ go-fetcher
 [![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io)
 [![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/)
 
-Go-fetcher is a library to retrieve IPLD prime nodes from IPFS using Bitswap.
+Go-fetcher is a library to retrieve IPLD prime nodes from IPFS using data exchange protocols
 
 ## Contribute
 

fetcher/fetcher.go

@@ -17,11 +17,15 @@ import (
  "github.com/ipld/go-ipld-prime/traversal/selector/builder"
 )
 
+// FetcherConfig defines a configuration object from which Fetcher instances are constructed
 type FetcherConfig struct {
  blockService blockservice.BlockService
+ NodeReifier ipld.NodeReifier
  PrototypeChooser traversal.LinkTargetNodePrototypeChooser
 }
 
+// Fetcher is an interface for reading from a dag. Reads may be local or remote, and may employ data exchange
+// protocols like graphsync and bitswap
 type Fetcher interface {
  // NodeMatching traverses a node graph starting with the provided node using the given selector and possibly crossing
  // block boundaries. Each matched node is passed as FetchResult to the callback. Errors returned from callback will
@@ -48,13 +52,15 @@ type fetcherSession struct {
  protoChooser traversal.LinkTargetNodePrototypeChooser
 }
 
+// FetchResult is a single node read as part of a dag operation called on a fetcher
 type FetchResult struct {
  Node ipld.Node
  Path ipld.Path
  LastBlockPath ipld.Path
  LastBlockLink ipld.Link
 }
 
+// FetchCallback is called for each node traversed during a fetch
 type FetchCallback func(result FetchResult) error
 
 // NewFetcherConfig creates a FetchConfig from which session may be created and nodes retrieved.
@@ -69,8 +75,14 @@ func NewFetcherConfig(blockService blockservice.BlockService) FetcherConfig {
 // The session ends when the provided context is canceled.
 func (fc FetcherConfig) NewSession(ctx context.Context) Fetcher {
  ls := cidlink.DefaultLinkSystem()
+ // while we may be loading blocks remotely, they are already hash verified by the time they load
+ // into ipld-prime
+ ls.TrustedStorage = true
  ls.StorageReadOpener = blockOpener(ctx, blockservice.NewSession(ctx, fc.blockService))
- return &fetcherSession{linkSystem: ls, protoChooser: fc.PrototypeChooser}
+ ls.NodeReifier = fc.NodeReifier
+
+ protoChooser := fc.PrototypeChooser
+ return &fetcherSession{linkSystem: ls, protoChooser: protoChooser}
 }
 
 // BlockOfType fetches a node graph of the provided type corresponding to single block by link.

fetcher/fetcher_test.go

@@ -290,3 +290,79 @@ func assertNodesInOrder(t *testing.T, results []fetcher.FetchResult, nodeCount i
 
  assert.Equal(t, nodeCount, len(results))
 }
+
+type selfLoader struct {
+ ipld.Node
+ ctx context.Context
+ ls *ipld.LinkSystem
+}
+
+func (sl *selfLoader) LookupByString(key string) (ipld.Node, error) {
+ nd, err := sl.Node.LookupByString(key)
+ if err != nil {
+ return nd, err
+ }
+ if nd.Kind() == ipld.Kind_Link {
+ lnk, _ := nd.AsLink()
+ nd, err = sl.ls.Load(ipld.LinkContext{Ctx: sl.ctx}, lnk, basicnode.Prototype.Any)
+ }
+ return nd, err
+}
+
+func TestNodeReification(t *testing.T) {
+ // demonstrates how to use the augment chooser to build an ADL that self loads its own nodes
+ block3, node3, link3 := testutil.EncodeBlock(fluent.MustBuildMap(basicnode.Prototype__Map{}, 1, func(na fluent.MapAssembler) {
+ na.AssembleEntry("three").AssignBool(true)
+ }))
+ block4, node4, link4 := testutil.EncodeBlock(fluent.MustBuildMap(basicnode.Prototype__Map{}, 1, func(na fluent.MapAssembler) {
+ na.AssembleEntry("four").AssignBool(true)
+ }))
+ block2, _, _ := testutil.EncodeBlock(fluent.MustBuildMap(basicnode.Prototype__Map{}, 2, func(na fluent.MapAssembler) {
+ na.AssembleEntry("link3").AssignLink(link3)
+ na.AssembleEntry("link4").AssignLink(link4)
+ }))
+
+ net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(0*time.Millisecond))
+ ig := testinstance.NewTestInstanceGenerator(net, nil, nil)
+ defer ig.Close()
+
+ peers := ig.Instances(2)
+ hasBlock := peers[0]
+ defer hasBlock.Exchange.Close()
+
+ err := hasBlock.Exchange.HasBlock(block2)
+ require.NoError(t, err)
+ err = hasBlock.Exchange.HasBlock(block3)
+ require.NoError(t, err)
+ err = hasBlock.Exchange.HasBlock(block4)
+ require.NoError(t, err)
+
+ wantsBlock := peers[1]
+ defer wantsBlock.Exchange.Close()
+
+ wantsGetter := blockservice.New(wantsBlock.Blockstore(), wantsBlock.Exchange)
+ fetcherConfig := fetcher.NewFetcherConfig(wantsGetter)
+ nodeReifier := func(lnkCtx ipld.LinkContext, nd ipld.Node, ls *ipld.LinkSystem) (ipld.Node, error) {
+ return &selfLoader{Node: nd, ctx: lnkCtx.Ctx, ls: ls}, nil
+ }
+ fetcherConfig.NodeReifier = nodeReifier
+ session := fetcherConfig.NewSession(context.Background())
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+
+ retrievedNode, err := fetcher.Block(ctx, session, cidlink.Link{Cid: block2.Cid()})
+ require.NoError(t, err)
+
+ // instead of getting links back, we automatically load the nodes
+
+ retrievedNode3, err := retrievedNode.LookupByString("link3")
+ require.NoError(t, err)
+ underlying3 := retrievedNode3.(*selfLoader).Node
+ assert.Equal(t, node3, underlying3)
+
+ retrievedNode4, err := retrievedNode.LookupByString("link4")
+ require.NoError(t, err)
+ underlying4 := retrievedNode4.(*selfLoader).Node
+ assert.Equal(t, node4, underlying4)
+
+}