introduce LinkSystem

CHANGELOG.md

@@ -39,7 +39,29 @@ Unreleased on master
 Changes here are on the master branch, but not in any tagged release yet.
 When a release tag is made, this block of bullet points will just slide down to the [Released Changes](#released-changes) section.
 
-- _nothing yet :)_
+- Change: linking has been significantly reworked, and now primarily works through the `ipld.LinkSystem` type.
+ - This is cool, because it makes a lot of things less circuitous. Previously, working with links was a complicated combination of Loader and Storer functions, the Link interface contained the Load method, it was just... complicated to figure out where to start. Now, the answer is simple and constant: "Start with LinkSystem". Clearer to use; clearer to document; and also coincidentally a lot clearer to develop for, internally.
+ - `Link.Load` -> `LinkSystem.Load` (or, new: `LinkSystem.Fill`, which lets you control memory allocation more explicitly).
+ - `LinkBuilder.Build` -> `LinkSystem.Store`.
+ - `LinkSystem.ComputeLink` is a new feature that prodices a Link without needing to store the data anywhere.
+ - The `ipld.Loader` function is now most analogous to `ipld.BlockReadOpener`. You now put it into use by assigning it to a `LinkLoader`'s `StorageReadOpener` field.
+ - The `ipld.Storer` function is now most analogous to `ipld.BlockWriteOpener`. You now put it into use by assigning it to a `LinkLoader`'s `StorageWriteOpener` field.
+ - 99% of the time, you'll probably start with `linking/cid.DefaultLinkSystem()`. You can assign to fields of this to customize it further, but it'll get you started with multihashes and multicodecs and all the behavior you expect when working with CIDs.
+ - (So, no -- the `cidlink` package hasn't gone anywhere. Hopefully it's a bit less obtrusive now, but it's still here.)
+ - The `traversal` package's `Config` struct now uses a `LinkSystem` instead of a `Loader` and `Storer` pair, as you would now probably expect.
+ - If you had code that was also previously passing around `Loader` and `Storer`, it's likely a similar pattern of change will be the right direction for that code.
+- Change: multicodec registration is now in the `go-ipld-prime/multicodec` package.
+ - Previously, this registry was in the `linking/cid` package. These things are now better decoupled.
+ - This wil require packages which register codecs to make some very small updates: e.g. `s/cidlink.RegisterMulticodecDecoder/multicodec.RegisterDecoder/`, and correspondingly, update the package imports at the top of the file.
+- New: some pre-made storage options (e.g. satisfying the `ipld.StorageReadOpener` and `ipld.StorageWriteOpener` function interfaces) have appeared! Find these in the `go-ipld-prime/storage` package.
+ - Currently this only includes a simple in-memory storage option. This may be useful for testing and examples, but probably not much else :)
+ - These are mostly intended to be illustrative. You should still expect to find better storage mechanisms in other repos.
+- Change: some function names in codec packages are ever-so-slightly updated. (They're verbs now, instead of nouns, which makes sense because they're functions. I have no idea what I was thinking with the previous naming. Sorry.)
+ - `s/dagjson.Decoder/dagjson.Decode/g`
+ - `s/dagjson.Decoder/dagjson.Encode/g`
+ - `s/dagcbor.Decoder/dagcbor.Decode/g`
+ - `s/dagcbor.Encoder/dagcbor.Encode/g`
+ - If you've only been using these indirectly, via their multicodec indicators, you won't have to update anything at all to account for this change.
 
 
 Released Changes

codec/api.go

@@ -1,13 +1,10 @@
 package codec
 
 import (
- "io"
-
  "github.com/ipld/go-ipld-prime"
 )
 
-// Encoder is the essential definition of a function that takes IPLD Data Model data in memory and serializes it.
-// IPLD Codecs are written by implementing this function interface (as well as (typically) a matched Decoder).
+// Encoder is defined in the root ipld package; this alias is just for documentation and discoverability.
 //
 // Encoder functions can be composed into an ipld.LinkSystem to provide
 // a "one stop shop" API for handling content addressable storage.
@@ -33,15 +30,12 @@ import (
 // in all scenarios that use codecs indirectly.
 // There is also no standard interface for such configurations: by nature,
 // if they exist at all, they vary per codec.
-type Encoder func(data ipld.Node, output io.Writer) error
+type Encoder = ipld.Encoder
 
-// Decoder is the essential definiton of a function that consumes serial data and unfurls it into IPLD Data Model-compatible in-memory representations.
-// IPLD Codecs are written by implementing this function interface (as well as (typically) a matched Encoder).
+// Decoder is defined in the root ipld package; this alias is just for documentation and discoverability.
 //
-// Decoder is the dual of Encoder.
-// Most of the documentation for the Encoder function interface
-// also applies wholesale to the Decoder interface.
-type Decoder func(into ipld.NodeAssembler, input io.Reader) error
+// Most of the documentation for Encoder also applies wholesale to the Decoder interface.
+type Decoder = ipld.Decoder
 
 type ErrBudgetExhausted struct{}
 

codec/dagcbor/multicodec.go

@@ -5,23 +5,22 @@ import (
 
  "github.com/polydawn/refmt/cbor"
 
- ipld "github.com/ipld/go-ipld-prime"
- cidlink "github.com/ipld/go-ipld-prime/linking/cid"
+ "github.com/ipld/go-ipld-prime"
+ "github.com/ipld/go-ipld-prime/multicodec"
 )
 
 var (
- _ cidlink.MulticodecDecoder = Decoder
- _ cidlink.MulticodecEncoder = Encoder
+ _ ipld.Decoder = Decode
+ _ ipld.Encoder = Encode
 )
 
 func init() {
- cidlink.RegisterMulticodecDecoder(0x71, Decoder)
- cidlink.RegisterMulticodecEncoder(0x71, Encoder)
+ multicodec.RegisterEncoder(0x71, Encode)
+ multicodec.RegisterDecoder(0x71, Decode)
 }
 
-func Decoder(na ipld.NodeAssembler, r io.Reader) error {
+func Decode(na ipld.NodeAssembler, r io.Reader) error {
  // Probe for a builtin fast path. Shortcut to that if possible.
- // (ipldcbor.NodeBuilder supports this, for example.)
  type detectFastPath interface {
  DecodeDagCbor(io.Reader) error
  }
@@ -32,9 +31,8 @@ func Decoder(na ipld.NodeAssembler, r io.Reader) error {
  return Unmarshal(na, cbor.NewDecoder(cbor.DecodeOptions{}, r))
 }
 
-func Encoder(n ipld.Node, w io.Writer) error {
+func Encode(n ipld.Node, w io.Writer) error {
  // Probe for a builtin fast path. Shortcut to that if possible.
- // (ipldcbor.Node supports this, for example.)
  type detectFastPath interface {
  EncodeDagCbor(io.Writer) error
  }

codec/dagcbor/roundtripCidlink_test.go

@@ -2,7 +2,6 @@ package dagcbor
 
 import (
  "bytes"
- "context"
  "io"
  "testing"
 
@@ -15,27 +14,26 @@ import (
 )
 
 func TestRoundtripCidlink(t *testing.T) {
- lb := cidlink.LinkBuilder{cid.Prefix{
+ lp := cidlink.LinkPrototype{cid.Prefix{
  Version: 1,
  Codec: 0x71,
- MhType: 0x17,
+ MhType: 0x13,
  MhLength: 4,
  }}
+ lsys := cidlink.DefaultLinkSystem()
 
  buf := bytes.Buffer{}
- lnk, err := lb.Build(context.Background(), ipld.LinkContext{}, n,
- func(ipld.LinkContext) (io.Writer, ipld.StoreCommitter, error) {
- return &buf, func(lnk ipld.Link) error { return nil }, nil
- },
- )
+ lsys.StorageWriteOpener = func(lnkCtx ipld.LinkContext) (io.Writer, ipld.BlockWriteCommitter, error) {
+ return &buf, func(lnk ipld.Link) error { return nil }, nil
+ }
+ lsys.StorageReadOpener = func(lnkCtx ipld.LinkContext, lnk ipld.Link) (io.Reader, error) {
+ return bytes.NewReader(buf.Bytes()), nil
+ }
+
+ lnk, err := lsys.Store(ipld.LinkContext{}, lp, n)
  Require(t, err, ShouldEqual, nil)
 
- nb := basicnode.Prototype__Any{}.NewBuilder()
- err = lnk.Load(context.Background(), ipld.LinkContext{}, nb,
- func(lnk ipld.Link, _ ipld.LinkContext) (io.Reader, error) {
- return bytes.NewReader(buf.Bytes()), nil
- },
- )
+ n2, err := lsys.Load(ipld.LinkContext{}, lnk, basicnode.Prototype.Any)
  Require(t, err, ShouldEqual, nil)
- Wish(t, nb.Build(), ShouldEqual, n)
+ Wish(t, n2, ShouldEqual, n)
 }

codec/dagcbor/roundtrip_test.go

@@ -2,16 +2,13 @@ package dagcbor
 
 import (
  "bytes"
- "context"
  "crypto/rand"
- "io"
  "strings"
  "testing"
 
  cid "github.com/ipfs/go-cid"
  . "github.com/warpfork/go-wish"
 
- ipld "github.com/ipld/go-ipld-prime"
  "github.com/ipld/go-ipld-prime/fluent"
  cidlink "github.com/ipld/go-ipld-prime/linking/cid"
  basicnode "github.com/ipld/go-ipld-prime/node/basic"
@@ -38,14 +35,14 @@ var serial = "\xa4eplainkolde stringcmap\xa2cone\x01ctwo\x02dlist\x82ethreedfour
 func TestRoundtrip(t *testing.T) {
  t.Run("encoding", func(t *testing.T) {
  var buf bytes.Buffer
- err := Encoder(n, &buf)
+ err := Encode(n, &buf)
  Require(t, err, ShouldEqual, nil)
  Wish(t, buf.String(), ShouldEqual, serial)
  })
  t.Run("decoding", func(t *testing.T) {
  buf := strings.NewReader(serial)
  nb := basicnode.Prototype__Map{}.NewBuilder()
- err := Decoder(nb, buf)
+ err := Decode(nb, buf)
  Require(t, err, ShouldEqual, nil)
  Wish(t, nb.Build(), ShouldEqual, n)
  })
@@ -57,33 +54,26 @@ func TestRoundtripScalar(t *testing.T) {
  simple := nb.Build()
  t.Run("encoding", func(t *testing.T) {
  var buf bytes.Buffer
- err := Encoder(simple, &buf)
+ err := Encode(simple, &buf)
  Require(t, err, ShouldEqual, nil)
  Wish(t, buf.String(), ShouldEqual, `japplesauce`)
  })
  t.Run("decoding", func(t *testing.T) {
  buf := strings.NewReader(`japplesauce`)
  nb := basicnode.Prototype__String{}.NewBuilder()
- err := Decoder(nb, buf)
+ err := Decode(nb, buf)
  Require(t, err, ShouldEqual, nil)
  Wish(t, nb.Build(), ShouldEqual, simple)
  })
 }
 
 func TestRoundtripLinksAndBytes(t *testing.T) {
- lb := cidlink.LinkBuilder{cid.Prefix{
+ lnk := cidlink.LinkPrototype{cid.Prefix{
  Version: 1,
  Codec: 0x71,
- MhType: 0x17,
+ MhType: 0x13,
  MhLength: 4,
- }}
- buf := bytes.Buffer{}
- lnk, err := lb.Build(context.Background(), ipld.LinkContext{}, n,
- func(ipld.LinkContext) (io.Writer, ipld.StoreCommitter, error) {
- return &buf, func(lnk ipld.Link) error { return nil }, nil
- },
- )
- Require(t, err, ShouldEqual, nil)
+ }}.BuildLink([]byte{1, 2, 3, 4}) // dummy value, content does not matter to this test.
 
  var linkByteNode = fluent.MustBuildMap(basicnode.Prototype__Map{}, 4, func(na fluent.MapAssembler) {
  nva := na.AssembleEntry("Link")
@@ -94,11 +84,11 @@ func TestRoundtripLinksAndBytes(t *testing.T) {
  nva.AssignBytes(bytes)
  })
 
- buf.Reset()
- err = Encoder(linkByteNode, &buf)
+ buf := bytes.Buffer{}
+ err := Encode(linkByteNode, &buf)
  Require(t, err, ShouldEqual, nil)
  nb := basicnode.Prototype__Map{}.NewBuilder()
- err = Decoder(nb, &buf)
+ err = Decode(nb, &buf)
  Require(t, err, ShouldEqual, nil)
  reconstructed := nb.Build()
  Wish(t, reconstructed, ShouldEqual, linkByteNode)

codec/dagcbor/unmarshal_test.go

@@ -14,28 +14,28 @@ func TestFunBlocks(t *testing.T) {
  // This fixture has a zero length link -- not even the multibase byte (which dag-cbor insists must be zero) is there.
  buf := strings.NewReader("\x8d\x8d\x97\xd8*@")
  nb := basicnode.Prototype.Any.NewBuilder()
- err := Decoder(nb, buf)
+ err := Decode(nb, buf)
  Require(t, err, ShouldEqual, ErrInvalidMultibase)
  })
  t.Run("fuzz001", func(t *testing.T) {
  // This fixture might cause an overly large allocation if you aren't careful to have resource budgets.
  buf := strings.NewReader("\x9a\xff000")
  nb := basicnode.Prototype.Any.NewBuilder()
- err := Decoder(nb, buf)
+ err := Decode(nb, buf)
  Require(t, err, ShouldEqual, ErrAllocationBudgetExceeded)
  })
  t.Run("fuzz002", func(t *testing.T) {
  // This fixture might cause an overly large allocation if you aren't careful to have resource budgets.
  buf := strings.NewReader("\x9f\x9f\x9f\x9f\x9f\x9f\x9f\x9f\x9f\x9f\x9f\x9f\x9f\x9f\x9f\x9f\x9f\x9f\x9f\x9f\x9a\xff000")
  nb := basicnode.Prototype.Any.NewBuilder()
- err := Decoder(nb, buf)
+ err := Decode(nb, buf)
  Require(t, err, ShouldEqual, ErrAllocationBudgetExceeded)
  })
  t.Run("fuzz003", func(t *testing.T) {
  // This fixture might cause an overly large allocation if you aren't careful to have resource budgets.
  buf := strings.NewReader("\x9f\x9f\x9f\x9f\x9f\x9f\x9f\xbb00000000")
  nb := basicnode.Prototype.Any.NewBuilder()
- err := Decoder(nb, buf)
+ err := Decode(nb, buf)
  Require(t, err, ShouldEqual, ErrAllocationBudgetExceeded)
  })
 }

codec/dagjson/multicodec.go

@@ -6,21 +6,21 @@ import (
 
  "github.com/polydawn/refmt/json"
 
- ipld "github.com/ipld/go-ipld-prime"
- cidlink "github.com/ipld/go-ipld-prime/linking/cid"
+ "github.com/ipld/go-ipld-prime"
+ "github.com/ipld/go-ipld-prime/multicodec"
 )
 
 var (
- _ cidlink.MulticodecDecoder = Decoder
- _ cidlink.MulticodecEncoder = Encoder
+ _ ipld.Decoder = Decode
+ _ ipld.Encoder = Encode
 )
 
 func init() {
- cidlink.RegisterMulticodecDecoder(0x0129, Decoder)
- cidlink.RegisterMulticodecEncoder(0x0129, Encoder)
+ multicodec.RegisterEncoder(0x0129, Encode)
+ multicodec.RegisterDecoder(0x0129, Decode)
 }
 
-func Decoder(na ipld.NodeAssembler, r io.Reader) error {
+func Decode(na ipld.NodeAssembler, r io.Reader) error {
  // Shell out directly to generic builder path.
  // (There's not really any fastpaths of note for json.)
  err := Unmarshal(na, json.NewDecoder(r))
@@ -52,7 +52,7 @@ func Decoder(na ipld.NodeAssembler, r io.Reader) error {
  return err
 }
 
-func Encoder(n ipld.Node, w io.Writer) error {
+func Encode(n ipld.Node, w io.Writer) error {
  // Shell out directly to generic inspection path.
  // (There's not really any fastpaths of note for json.)
  // Write another function if you need to tune encoding options about whitespace.

codec/dagjson/roundtripCidlink_test.go

@@ -2,9 +2,7 @@ package dagjson
 
 import (
  "bytes"
- "context"
  "io"
- "io/ioutil"
  "strings"
  "testing"
 
@@ -17,29 +15,28 @@ import (
 )
 
 func TestRoundtripCidlink(t *testing.T) {
- lb := cidlink.LinkBuilder{cid.Prefix{
+ lp := cidlink.LinkPrototype{cid.Prefix{
  Version: 1,
  Codec: 0x0129,
- MhType: 0x17,
+ MhType: 0x13,
  MhLength: 4,
  }}
+ lsys := cidlink.DefaultLinkSystem()
 
  buf := bytes.Buffer{}
- lnk, err := lb.Build(context.Background(), ipld.LinkContext{}, n,
- func(ipld.LinkContext) (io.Writer, ipld.StoreCommitter, error) {
- return &buf, func(lnk ipld.Link) error { return nil }, nil
- },
- )
+ lsys.StorageWriteOpener = func(lnkCtx ipld.LinkContext) (io.Writer, ipld.BlockWriteCommitter, error) {
+ return &buf, func(lnk ipld.Link) error { return nil }, nil
+ }
+ lsys.StorageReadOpener = func(lnkCtx ipld.LinkContext, lnk ipld.Link) (io.Reader, error) {
+ return bytes.NewReader(buf.Bytes()), nil
+ }
+
+ lnk, err := lsys.Store(ipld.LinkContext{}, lp, n)
  Require(t, err, ShouldEqual, nil)
 
- nb := basicnode.Prototype__Any{}.NewBuilder()
- err = lnk.Load(context.Background(), ipld.LinkContext{}, nb,
- func(lnk ipld.Link, _ ipld.LinkContext) (io.Reader, error) {
- return bytes.NewReader(buf.Bytes()), nil
- },
- )
+ n2, err := lsys.Load(ipld.LinkContext{}, lnk, basicnode.Prototype.Any)
  Require(t, err, ShouldEqual, nil)
- Wish(t, nb.Build(), ShouldEqual, n)
+ Wish(t, n2, ShouldEqual, n)
 }
 
 // Make sure that a map that *almost* looks like a link is handled safely.
@@ -48,24 +45,19 @@ func TestRoundtripCidlink(t *testing.T) {
 // tokens have to be reprocessed before a recursion that find a real link appears.
 func TestUnmarshalTrickyMapContainingLink(t *testing.T) {
  // Create a link; don't particularly care about its contents.
- lnk, err := cidlink.LinkBuilder{cid.Prefix{
+ lnk := cidlink.LinkPrototype{cid.Prefix{
  Version: 1,
- Codec: 0x0129,
- MhType: 0x17,
+ Codec: 0x71,
+ MhType: 0x13,
  MhLength: 4,
- }}.Build(context.Background(), ipld.LinkContext{}, n,
- func(ipld.LinkContext) (io.Writer, ipld.StoreCommitter, error) {
- return ioutil.Discard, func(lnk ipld.Link) error { return nil }, nil
- },
- )
- Require(t, err, ShouldEqual, nil)
+ }}.BuildLink([]byte{1, 2, 3, 4}) // dummy value, content does not matter to this test.
 
  // Compose the tricky corpus. (lnk.String "happens" to work here, although this isn't recommended or correct in general.)
  tricky := `{"/":{"/":"` + lnk.String() + `"}}`
 
  // Unmarshal. Hopefully we get a map with a link in it.
  nb := basicnode.Prototype__Any{}.NewBuilder()
- err = Decoder(nb, strings.NewReader(tricky))
+ err := Decode(nb, strings.NewReader(tricky))
  Require(t, err, ShouldEqual, nil)
  n := nb.Build()
  Wish(t, n.Kind(), ShouldEqual, ipld.Kind_Map)