feat(fetcher): add on demand prototype chooser augmentation

README.md

@@ -4,7 +4,7 @@ go-fetcher
 [![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io)
 [![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/)
 
-Go-fetcher is a library to retrieve IPLD prime nodes from IPFS using Bitswap.
+Go-fetcher is a library to retrieve IPLD prime nodes from IPFS using data exchange protocols
 
 ## Contribute
 

fetcher.go

@@ -17,11 +17,19 @@ import (
  "github.com/ipld/go-ipld-prime/traversal/selector/builder"
 )
 
+// AugmentChooserFunc is a function that can augment a prototype chooser at the time the Fetcher is initialized,
+// which is given the linksystem the fetcher itself will use
+type AugmentChooserFunc func(*ipld.LinkSystem, traversal.LinkTargetNodePrototypeChooser) traversal.LinkTargetNodePrototypeChooser
+
+// FetcherConfig defines a configuration object from which Fetcher instances are constructed
 type FetcherConfig struct {
  blockService blockservice.BlockService
+ AugmentChooser AugmentChooserFunc
  PrototypeChooser traversal.LinkTargetNodePrototypeChooser
 }
 
+// Fetcher is an interface for reading from a dag. Reads may be local or remote, and may employ data exchange
+// protocols like graphsync and bitswap
 type Fetcher interface {
  // NodeMatching traverses a node graph starting with the provided node using the given selector and possibly crossing
  // block boundaries. Each matched node is passed as FetchResult to the callback. Errors returned from callback will
@@ -48,13 +56,15 @@ type fetcherSession struct {
  protoChooser traversal.LinkTargetNodePrototypeChooser
 }
 
+// FetchResult is a single node read as part of a dag operation called on a fetcher
 type FetchResult struct {
  Node ipld.Node
  Path ipld.Path
  LastBlockPath ipld.Path
  LastBlockLink ipld.Link
 }
 
+// FetchCallback is called for each node traversed during a fetch
 type FetchCallback func(result FetchResult) error
 
 // NewFetcherConfig creates a FetchConfig from which session may be created and nodes retrieved.
@@ -70,7 +80,11 @@ func NewFetcherConfig(blockService blockservice.BlockService) FetcherConfig {
 func (fc FetcherConfig) NewSession(ctx context.Context) Fetcher {
  ls := cidlink.DefaultLinkSystem()
  ls.StorageReadOpener = blockOpener(ctx, blockservice.NewSession(ctx, fc.blockService))
- return &fetcherSession{linkSystem: ls, protoChooser: fc.PrototypeChooser}
+ protoChooser := fc.PrototypeChooser
+ if fc.AugmentChooser != nil {
+ protoChooser = fc.AugmentChooser(&ls, protoChooser)
+ }
+ return &fetcherSession{linkSystem: ls, protoChooser: protoChooser}
 }
 
 // BlockOfType fetches a node graph of the provided type corresponding to single block by link.

fetcher_test.go

@@ -6,6 +6,7 @@ import (
  "testing"
  "time"
 
+ "github.com/ipld/go-ipld-prime/traversal"
  "github.com/ipld/go-ipld-prime/traversal/selector"
  "github.com/ipld/go-ipld-prime/traversal/selector/builder"
 
@@ -290,3 +291,104 @@ func assertNodesInOrder(t *testing.T, results []fetcher.FetchResult, nodeCount i
 
  assert.Equal(t, nodeCount, len(results))
 }
+
+type selfLoader struct {
+ ipld.Node
+ ctx context.Context
+ ls *ipld.LinkSystem
+}
+
+func (sl *selfLoader) LookupByString(key string) (ipld.Node, error) {
+ nd, err := sl.Node.LookupByString(key)
+ if err != nil {
+ return nd, err
+ }
+ if nd.Kind() == ipld.Kind_Link {
+ lnk, _ := nd.AsLink()
+ nd, err = sl.ls.Load(ipld.LinkContext{Ctx: sl.ctx}, lnk, basicnode.Prototype.Any)
+ }
+ return nd, err
+}
+
+type selfLoadPrototype struct {
+ ctx context.Context
+ ls *ipld.LinkSystem
+ basePrototype ipld.NodePrototype
+}
+
+func (slp *selfLoadPrototype) NewBuilder() ipld.NodeBuilder {
+ return &selfLoadBuilder{ctx: slp.ctx, NodeBuilder: slp.basePrototype.NewBuilder(), ls: slp.ls}
+}
+
+type selfLoadBuilder struct {
+ ctx context.Context
+ ipld.NodeBuilder
+ ls *ipld.LinkSystem
+}
+
+func (slb *selfLoadBuilder) Build() ipld.Node {
+ nd := slb.NodeBuilder.Build()
+ return &selfLoader{nd, slb.ctx, slb.ls}
+}
+
+func TestChooserAugmentation(t *testing.T) {
+ // demonstrates how to use the augment chooser to build an ADL that self loads its own nodes
+ block3, node3, link3 := testutil.EncodeBlock(fluent.MustBuildMap(basicnode.Prototype__Map{}, 1, func(na fluent.MapAssembler) {
+ na.AssembleEntry("three").AssignBool(true)
+ }))
+ block4, node4, link4 := testutil.EncodeBlock(fluent.MustBuildMap(basicnode.Prototype__Map{}, 1, func(na fluent.MapAssembler) {
+ na.AssembleEntry("four").AssignBool(true)
+ }))
+ block2, _, _ := testutil.EncodeBlock(fluent.MustBuildMap(basicnode.Prototype__Map{}, 2, func(na fluent.MapAssembler) {
+ na.AssembleEntry("link3").AssignLink(link3)
+ na.AssembleEntry("link4").AssignLink(link4)
+ }))
+
+ net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(0*time.Millisecond))
+ ig := testinstance.NewTestInstanceGenerator(net, nil, nil)
+ defer ig.Close()
+
+ peers := ig.Instances(2)
+ hasBlock := peers[0]
+ defer hasBlock.Exchange.Close()
+
+ err := hasBlock.Exchange.HasBlock(block2)
+ require.NoError(t, err)
+ err = hasBlock.Exchange.HasBlock(block3)
+ require.NoError(t, err)
+ err = hasBlock.Exchange.HasBlock(block4)
+ require.NoError(t, err)
+
+ wantsBlock := peers[1]
+ defer wantsBlock.Exchange.Close()
+
+ wantsGetter := blockservice.New(wantsBlock.Blockstore(), wantsBlock.Exchange)
+ fetcherConfig := fetcher.NewFetcherConfig(wantsGetter)
+ augmentChooser := func(ls *ipld.LinkSystem, base traversal.LinkTargetNodePrototypeChooser) traversal.LinkTargetNodePrototypeChooser {
+ return func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) {
+ np, err := base(lnk, lnkCtx)
+ if err != nil {
+ return np, err
+ }
+ return &selfLoadPrototype{ctx: lnkCtx.Ctx, ls: ls, basePrototype: np}, nil
+ }
+ }
+ fetcherConfig.AugmentChooser = augmentChooser
+ session := fetcherConfig.NewSession(context.Background())
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+
+ retrievedNode, err := fetcher.Block(ctx, session, cidlink.Link{Cid: block2.Cid()})
+ require.NoError(t, err)
+
+ // instead of getting links back, we automatically load the nodes
+
+ retrievedNode3, err := retrievedNode.LookupByString("link3")
+ require.NoError(t, err)
+ assert.Equal(t, node3, retrievedNode3)
+
+ retrievedNode4, err := retrievedNode.LookupByString("link4")
+ require.NoError(t, err)
+ assert.Equal(t, node4, retrievedNode4)
+
+}

go.mod

@@ -11,6 +11,6 @@ require (
  github.com/ipfs/go-ipfs-delay v0.0.1
  github.com/ipfs/go-ipfs-routing v0.1.0
  github.com/ipld/go-codec-dagpb v1.2.0
- github.com/ipld/go-ipld-prime v0.9.0
+ github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db
  github.com/stretchr/testify v1.6.1
 )

go.sum

@@ -193,6 +193,8 @@ github.com/ipld/go-codec-dagpb v1.2.0 h1:2umV7ud8HBMkRuJgd8gXw95cLhwmcYrihS3cQEy
 github.com/ipld/go-codec-dagpb v1.2.0/go.mod h1:6nBN7X7h8EOsEejZGqC7tej5drsdBAXbMHyBT+Fne5s=
 github.com/ipld/go-ipld-prime v0.9.0 h1:N2OjJMb+fhyFPwPnVvJcWU/NsumP8etal+d2v3G4eww=
 github.com/ipld/go-ipld-prime v0.9.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8=
+github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db h1:kFwGn8rXa/Z31ev1OFNQsYeNKNCdifnTPl/NvPy5L38=
+github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8=
 github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=
 github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
 github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus=