Skip to content

Commit

Permalink
examples/rapido: add example on how to use rapide
Browse files Browse the repository at this point in the history
  • Loading branch information
Jorropo committed Feb 3, 2023
1 parent 021b3c9 commit 143cd01
Show file tree
Hide file tree
Showing 4 changed files with 97 additions and 3 deletions.
2 changes: 1 addition & 1 deletion examples/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ require (
github.com/ipfs/go-unixfs v0.4.2
github.com/ipfs/go-unixfsnode v1.5.1
github.com/ipfs/interface-go-ipfs-core v0.10.0
github.com/ipld/go-car v0.6.0
github.com/ipld/go-car/v2 v2.6.0
github.com/ipld/go-codec-dagpb v1.5.0
github.com/ipld/go-ipld-prime v0.19.0
Expand Down Expand Up @@ -52,7 +53,6 @@ require (
github.com/ipfs/go-log/v2 v2.5.1 // indirect
github.com/ipfs/go-metrics-interface v0.0.1 // indirect
github.com/ipfs/go-verifcid v0.0.2 // indirect
github.com/ipld/go-car v0.5.0 // indirect
github.com/jbenet/goprocess v0.1.4 // indirect
github.com/klauspost/cpuid/v2 v2.2.3 // indirect
github.com/libp2p/go-buffer-pool v0.1.0 // indirect
Expand Down
4 changes: 2 additions & 2 deletions examples/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -463,8 +463,8 @@ github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvT
github.com/ipfs/go-verifcid v0.0.2/go.mod h1:40cD9x1y4OWnFXbLNJYRe7MpNvWlMn3LZAG5Wb4xnPU=
github.com/ipfs/interface-go-ipfs-core v0.10.0 h1:b/psL1oqJcySdQAsIBfW5ZJJkOAsYlhWtC0/Qvr4WiM=
github.com/ipfs/interface-go-ipfs-core v0.10.0/go.mod h1:F3EcmDy53GFkF0H3iEJpfJC320fZ/4G60eftnItrrJ0=
github.com/ipld/go-car v0.5.0 h1:kcCEa3CvYMs0iE5BzD5sV7O2EwMiCIp3uF8tA6APQT8=
github.com/ipld/go-car v0.5.0/go.mod h1:ppiN5GWpjOZU9PgpAZ9HbZd9ZgSpwPMr48fGRJOWmvE=
github.com/ipld/go-car v0.6.0 h1:d5QrGLnHAxiNLHor+DKGrLdqnM0dQJh2whfSXRDq6J0=
github.com/ipld/go-car v0.6.0/go.mod h1:tBrW1XZ3L2XipLxA69RnTVGW3rve6VX4TbaTYkq8aEA=
github.com/ipld/go-car/v2 v2.6.0 h1:UTJmJ99nxgYpPueoaZ1GrFQtUDe9QnVLlHYTNDSDb90=
github.com/ipld/go-car/v2 v2.6.0/go.mod h1:qoqfgPnQYcaAYcfphctffdaNWJIWBR2QN4pjuKUtgao=
github.com/ipld/go-codec-dagpb v1.3.0/go.mod h1:ga4JTU3abYApDC3pZ00BC2RSvC3qfBb9MSJkMLSwnhA=
Expand Down
11 changes: 11 additions & 0 deletions examples/rapido/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# RAPIDO

RAPIDO is an example CLI which download an unordered `.car` for some unixfs data from multiple gateways using [`rapide`](../../rapide).

This code is not maintained up to "production ready" standars, this is an example demonstrating how to use RAPIDE.

## Usage

```console
$ rapido -gw https://ipfs.io/ipfs/ -gw https://strn.pl/ipfs/ QmT2EHPdRvUDxiuZBbYg5ZHy1f8L6MY1HxD45zHycLobMJ | pv > 2023-02-03.ipfs.io.car
```
83 changes: 83 additions & 0 deletions examples/rapido/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
package main

import (
"context"
"flag"
"fmt"
"os"
"strings"

"github.com/ipfs/go-cid"
"github.com/ipfs/go-libipfs/ipsl/unixfs"
"github.com/ipfs/go-libipfs/rapide"
"github.com/ipfs/go-libipfs/rapide/gateway"
"github.com/ipld/go-car"
"github.com/ipld/go-car/util"
)

type stringFlagSlice []string

func (sfs *stringFlagSlice) String() string {
return strings.Join(*sfs, ",")
}

func (sfs *stringFlagSlice) Set(s string) error {
*sfs = append(*sfs, s)
return nil
}

func main() {
err := mainRet()
if err != nil {
os.Stderr.WriteString(err.Error())
os.Exit(1)
}
os.Exit(0)
}

func mainRet() error {
// Argument Parsing
var gateways stringFlagSlice
flag.Var(&gateways, "gw", `Set once to add a gateway, setable multiple times to use multiple gateways. Format expected is "-gw https://ipfs.io/ipfs/" for example.`)
flag.Parse()

cidStrs := flag.Args()
if len(cidStrs) != 1 {
return fmt.Errorf("expected one CID as positional argument; got %d", len(cidStrs))
}

root, err := cid.Decode(cidStrs[0])
if err != nil {
return fmt.Errorf("decoding CID: %w", err)
}

// Setup header for the output car
err = car.WriteHeader(&car.CarHeader{
Roots: []cid.Cid{root},
Version: 1,
}, os.Stdout)
if err != nil {
return fmt.Errorf("writing car header: %w", err)
}

// configure rapide
downloaders := make([]rapide.ServerDrivenDownloader, len(gateways)) // create a slice holding our multiple gateways
for i, g := range gateways {
downloaders[i] = gateway.Gateway{PathName: g} // create a gateway protocol implementation
}
client := rapide.Client{ServerDrivenDownloaders: downloaders}

// do request and iterate over the resulting blocks, rapide.(*Client).Get returns a channel
for maybeBlock := range client.Get(context.Background(), root, unixfs.Everything()) {
block, err := maybeBlock.Get() // block or error ?
if err != nil {
return fmt.Errorf("downloading: %w", err)
}
err = util.LdWrite(os.Stdout, block.Cid().Bytes(), block.RawData()) // write to the output car
if err != nil {
return fmt.Errorf("writing to output car: %w", err)
}
}

return nil
}

0 comments on commit 143cd01

Please sign in to comment.