Skip to content

Commit

Permalink
initial code
Browse files Browse the repository at this point in the history
  • Loading branch information
@aschmahmann
aschmahmann committed Nov 4, 2022
1 parent 6a1eab4 commit 5179c98
Show file tree
Hide file tree
Showing 12 changed files with 2,809 additions and 0 deletions.
53 changes: 53 additions & 0 deletions bswap.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
package mdinc

import (
"bytes"
"context"
"io"

bsclient "github.com/ipfs/go-bitswap/client"
"github.com/ipfs/go-bitswap/client/sessioniface"
bsnet "github.com/ipfs/go-bitswap/network"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-datastore"
bstore "github.com/ipfs/go-ipfs-blockstore"
"github.com/ipld/go-ipld-prime/datamodel"
"github.com/ipld/go-ipld-prime/linking"
cidlink "github.com/ipld/go-ipld-prime/linking/cid"
rhelpers "github.com/libp2p/go-libp2p-routing-helpers"
"github.com/libp2p/go-libp2p/core/host"
)

type addRemoveCid struct {
key cid.Cid
add bool
}

func (a addRemoveCid) IsAdd() bool {
return a.add
}

func (a addRemoveCid) Key() cid.Cid {
return a.key
}

var _ sessioniface.AddRemoveCid = (*addRemoveCid)(nil)

func bitswapBlockLoader(ctx context.Context, h host.Host) Loader {
n := bsnet.NewFromIpfsHost(h, rhelpers.Null{})
bs := bstore.NewBlockstore(datastore.NewMapDatastore())
c := bsclient.New(ctx, n, bs)
n.Start(c)
s := c.NewSession(ctx)

lsys := cidlink.DefaultLinkSystem()
lsys.StorageReadOpener = func(linkContext linking.LinkContext, link datamodel.Link) (io.Reader, error) {
blk, err := s.GetBlock(linkContext.Ctx, link.(cidlink.Link).Cid)
if err != nil {
return nil, err
}
return bytes.NewReader(blk.RawData()), nil
}
ld := NewBlockLoader(lsys, 100)
return ld
}
300 changes: 300 additions & 0 deletions cmd/mdinc.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,300 @@
package main

import (
"bytes"
"errors"
"fmt"
"io"
"log"
"os"

"github.com/urfave/cli/v2"

bsclient "github.com/ipfs/go-bitswap/client"
"github.com/ipfs/go-bitswap/client/sessioniface"
bsnet "github.com/ipfs/go-bitswap/network"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-datastore"
dssync "github.com/ipfs/go-datastore/sync"
drc "github.com/ipfs/go-delegated-routing/client"
drp "github.com/ipfs/go-delegated-routing/gen/proto"
blockstore "github.com/ipfs/go-ipfs-blockstore"
carbs "github.com/ipld/go-car/v2/blockstore"
"github.com/ipld/go-ipld-prime/datamodel"
"github.com/ipld/go-ipld-prime/linking"
cidlink "github.com/ipld/go-ipld-prime/linking/cid"
"github.com/libp2p/go-libp2p"
rhelpers "github.com/libp2p/go-libp2p-routing-helpers"
"github.com/libp2p/go-libp2p/core/peer"
"github.com/libp2p/go-libp2p/p2p/muxer/mplex"
"github.com/multiformats/go-multibase"
"github.com/multiformats/go-multihash"

"github.com/aschmahmann/mdinc"
)

func main() {
app := &cli.App{
Name: "mdinc",
Usage: "tools for working with incrementally verifiable large merkle-damgard blocks",
Commands: []*cli.Command{
{
Name: "ingest",
Usage: "<filepath> <outputpath>",
Description: "Take a file and turn it into a CAR file with both the file data and the proof information required for incremental verifiability. " +
"Returns the digest along with the CID of the proof. Currently only SHA2-256 is supported.",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "json",
Usage: "output data as JSON",
Value: false,
},
},
Action: func(ctx *cli.Context) error {
if ctx.NArg() != 2 {
return errors.New("invalid number of arguments")
}
fpath := ctx.Args().Get(0)
opath := ctx.Args().Get(1)

finput, err := os.Open(fpath)
if err != nil {
return err
}

fileDigest, proofRoot, err := mdinc.OutputFile(opath, finput, true, true)
if err != nil {
return err
}

fileDigestString, err := multibase.Encode(multibase.Base16, fileDigest)
if err != nil {
return err
}

if ctx.Bool("json") {
fmt.Printf("{ \"filehash\" : %q, \"proofCID\" : %q }\n", fileDigestString, proofRoot)
} else {
fmt.Printf("filehash %v, proofCID %v\n", fileDigestString, proofRoot)
}
return nil
},
},
{
Name: "verify",
Usage: "<multibase-multihash> <car-path> [proofCID]",
Description: "Takes a multihash and a CAR and attempts to verify that the data with that hash is in the CAR. " +
"The proof CID must either be the only root in the CAR file or passed explicitly.",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "speedy",
Value: false,
Usage: "Enables using the more complex, but faster verification scheme",
},
},
Action: func(ctx *cli.Context) error {
if ctx.NArg() > 3 || ctx.NArg() < 2 {
return errors.New("invalid number of arguments")
}
mhStr := ctx.Args().Get(0)
carPath := ctx.Args().Get(1)
_, mhBytes, err := multibase.Decode(mhStr)
if err != nil {
return err
}
mh, err := multihash.Cast(mhBytes)
if err != nil {
return err
}

bs, err := carbs.OpenReadOnly(carPath)
if err != nil {
return err
}

var proof cid.Cid
if ctx.NArg() == 3 {
cidStr := ctx.Args().Get(2)
proof, err = cid.Decode(cidStr)
if err != nil {
return err
}
} else {
roots, err := bs.Roots()
if err != nil {
return err
}
if len(roots) != 1 {
return fmt.Errorf("the proof root must be passed unless there is a single root in the CAR file")
}
proof = roots[0]
}

lsys := cidlink.DefaultLinkSystem()
lsys.StorageReadOpener = func(ctx linking.LinkContext, link datamodel.Link) (io.Reader, error) {
cl := link.(cidlink.Link)
blk, err := bs.Get(ctx.Ctx, cl.Cid)
if err != nil {
return nil, err
}
return bytes.NewReader(blk.RawData()), nil
}

if ctx.Bool("speedy") {
bs := blockstore.NewBlockstore(dssync.MutexWrap(datastore.NewMapDatastore()))
ld := mdinc.NewBlockLoader(lsys, 100)
_, err := mdinc.SpeedyVerify(ctx.Context, mh, proof, ld, bs)
return err
}

return mdinc.Verify(mh, proof, lsys)
},
},
{
Name: "download",
Usage: "<multibase-multihash> <proofCID> <multiaddr>",
Description: "Download the data associated with multihash using the given proof CID from a particular multiaddr. The target must speak the Bitswap protocol.",
Flags: []cli.Flag{
&cli.PathFlag{
Name: "output",
Usage: "output file path. if not defined will print to stdout",
Required: false,
Aliases: []string{"o"},
},
},
Action: func(ctx *cli.Context) error {
if ctx.NArg() != 3 {
return errors.New("invalid number of arguments")
}
mhStr := ctx.Args().Get(0)
cidStr := ctx.Args().Get(1)
maStr := ctx.Args().Get(2)
_, mhBytes, err := multibase.Decode(mhStr)
if err != nil {
return err
}
mh, err := multihash.Cast(mhBytes)
if err != nil {
return err
}

proof, err := cid.Decode(cidStr)
if err != nil {
return err
}

ai, err := peer.AddrInfoFromString(maStr)
if err != nil {
return err
}
h, err := libp2p.New(libp2p.DefaultMuxers, libp2p.Muxer("/mplex/6.7.0", mplex.DefaultTransport))
if err != nil {
return err
}

n := bsnet.NewFromIpfsHost(h, rhelpers.Null{})
// Note: this blockstore is largely irrelevant and seems to mostly be for metrics collection
bsclientBlockstore := blockstore.NewBlockstore(datastore.NewMapDatastore())
c := bsclient.New(ctx.Context, n, bsclientBlockstore)
n.Start(c)
s := c.NewSession(ctx.Context)
cf, ok := s.(sessioniface.ChannelFetcher)
if !ok {
return fmt.Errorf("session not a channel fetcher")
}
if err != nil {
return err
}
if err := h.Connect(ctx.Context, *ai); err != nil {
return err
}

bs := blockstore.NewBlockstore(dssync.MutexWrap(datastore.NewMapDatastore()))
data, err := mdinc.SpeedyVerify(ctx.Context, mh, proof, cf, bs)
if err != nil {
return err
}

if outputPath := ctx.Path("output"); outputPath != "" {
return os.WriteFile(outputPath, data, os.ModePerm)
}
if _, err := os.Stdout.Write(data); err != nil {
return err
}

return nil
},
},
{
Name: "download-with-discovery",
Usage: "<multibase-multihash> <router-url>",
Description: "Download the data associated with multihash using the given router for discovery/content routing. " +
"The router must be a Reframe implementation that supports returning proofs for large blocks " +
"and the targets that have the data must speak the Bitswap protocol.",
Flags: []cli.Flag{
&cli.PathFlag{
Name: "output",
Usage: "output file path. if not defined will print to stdout",
Required: false,
Aliases: []string{"o"},
},
},
Action: func(ctx *cli.Context) error {
if ctx.NArg() != 2 {
return errors.New("invalid number of arguments")
}
mhStr := ctx.Args().Get(0)
routerUrl := ctx.Args().Get(1)
_, mhBytes, err := multibase.Decode(mhStr)
if err != nil {
return err
}
mh, err := multihash.Cast(mhBytes)
if err != nil {
return err
}

h, err := libp2p.New(libp2p.DefaultMuxers, libp2p.Muxer("/mplex/6.7.0", mplex.DefaultTransport))
if err != nil {
return err
}

drpclient, err := drp.New_DelegatedRouting_Client(routerUrl)
if err != nil {
return err
}

drclient, err := drc.NewClient(drpclient, nil, nil)
if err != nil {
return err
}
dr := drc.NewContentRoutingClient(drclient)

n := bsnet.NewFromIpfsHost(h, dr)
bs := blockstore.NewBlockstore(datastore.NewMapDatastore())
downloadClient := bsclient.New(ctx.Context, n, bs)
n.Start(downloadClient)

data, err := mdinc.Download2(ctx.Context, mh, routerUrl, h, downloadClient)
if err != nil {
return err
}

if outputPath := ctx.Path("output"); outputPath != "" {
return os.WriteFile(outputPath, data, os.ModePerm)
}
if _, err := os.Stdout.Write(data); err != nil {
return err
}

return nil
},
},
},
}

err := app.Run(os.Args)
if err != nil {
log.Fatal(err)
}
}
Loading

0 comments on commit 5179c98

Please sign in to comment.