Skip to content

Commit

Permalink
libp2p retrieval transports endpoint (#723)
Browse files Browse the repository at this point in the history
* feat: libp2p retrieval transports endpoint

* refactor: transports protocol can have multiple multiaddresses

* feat: boost provider retrieval-transports CLI command

* docs: add known protocols to transports ipld schema
  • Loading branch information
dirkmc authored Aug 24, 2022
1 parent 6b637b9 commit edfaf9f
Show file tree
Hide file tree
Showing 11 changed files with 428 additions and 6 deletions.
109 changes: 109 additions & 0 deletions cmd/boost/provider_cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,16 @@ import (
"github.com/filecoin-project/boost/cli/ctxutil"
clinode "github.com/filecoin-project/boost/cli/node"
"github.com/filecoin-project/boost/cmd"
"github.com/filecoin-project/boost/retrievalmarket/lp2pimpl"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
"github.com/filecoin-project/go-fil-markets/storagemarket/network"
// TODO: This multiaddr util library should probably live in its own repo
multiaddrutil "github.com/filecoin-project/go-legs/httpsync/multiaddr"
"github.com/filecoin-project/lotus/chain/types"
lcli "github.com/filecoin-project/lotus/cli"
"github.com/ipfs/go-cid"
"github.com/multiformats/go-multiaddr"
"github.com/urfave/cli/v2"
)

Expand All @@ -30,6 +34,7 @@ var providerCmd = &cli.Command{
libp2pInfoCmd,
storageAskCmd,
retrievalAskCmd,
retrievalTransportsCmd,
},
}

Expand Down Expand Up @@ -294,3 +299,107 @@ var retrievalAskCmd = &cli.Command{
return nil
},
}

var retrievalTransportsCmd = &cli.Command{
Name: "retrieval-transports",
Usage: "Query a storage provider's available retrieval transports (libp2p, http, etc)",
ArgsUsage: "[provider]",
Action: func(cctx *cli.Context) error {
ctx := bcli.ReqContext(cctx)

afmt := NewAppFmt(cctx.App)
if cctx.NArg() != 1 {
afmt.Println("Usage: retrieval-transports [provider]")
return nil
}

n, err := clinode.Setup(cctx.String(cmd.FlagRepo.Name))
if err != nil {
return err
}

api, closer, err := lcli.GetGatewayAPI(cctx)
if err != nil {
return fmt.Errorf("cant setup gateway connection: %w", err)
}
defer closer()

maddr, err := address.NewFromString(cctx.Args().First())
if err != nil {
return err
}

addrInfo, err := cmd.GetAddrInfo(ctx, api, maddr)
if err != nil {
return err
}

log.Debugw("found storage provider", "id", addrInfo.ID, "multiaddrs", addrInfo.Addrs, "addr", maddr)

if err := n.Host.Connect(ctx, *addrInfo); err != nil {
return fmt.Errorf("failed to connect to peer %s: %w", addrInfo.ID, err)
}

// Send the query to the Storage Provider
client := lp2pimpl.NewTransportsClient(n.Host)
resp, err := client.SendQuery(ctx, addrInfo.ID)
if err != nil {
return fmt.Errorf("failed to fetch transports from peer %s: %w", addrInfo.ID, err)
}

if cctx.Bool("json") {
type addr struct {
Multiaddr string `json:"multiaddr"`
Address string `json:"address,omitempty"`
}
json := make(map[string]interface{}, len(resp.Protocols))
for _, p := range resp.Protocols {
addrs := make([]addr, 0, len(p.Addresses))
for _, ma := range p.Addresses {
// Get the multiaddress, and also try to get the address
// in the protocol's native format (eg URL format for
// http protocol)
addrs = append(addrs, addr{
Multiaddr: ma.String(),
Address: multiaddrToNative(p.Name, ma),
})
}
json[p.Name] = addrs
}
return cmd.PrintJson(json)
}

if len(resp.Protocols) == 0 {
afmt.Println("No available retrieval protocols")
return nil
}
for _, p := range resp.Protocols {
afmt.Println(p.Name)
for _, a := range p.Addresses {
// Output the multiaddress
afmt.Println(" " + a.String())
// Try to get the address in the protocol's native format
// (eg URL format for http protocol)
nativeFmt := multiaddrToNative(p.Name, a)
if nativeFmt != "" {
afmt.Println(" " + nativeFmt)
}
}
}

return nil
},
}

func multiaddrToNative(proto string, ma multiaddr.Multiaddr) string {
switch proto {
case "http", "https":
u, err := multiaddrutil.ToURL(ma)
if err != nil {
return ""
}
return u.String()
}

return ""
}
45 changes: 45 additions & 0 deletions cmd/boost/provider_cmd_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
package main

import (
"github.com/multiformats/go-multiaddr"
"github.com/stretchr/testify/require"
"testing"
)

func TestMultiaddrToNative(t *testing.T) {
testCases := []struct {
name string
proto string
ma string
expected string
}{{
name: "http",
proto: "http",
ma: "/dns/foo.com/http",
expected: "http://foo.com",
}, {
name: "http IP 4 address",
proto: "http",
ma: "/ip4/192.168.0.1/tcp/80/http",
expected: "http://192.168.0.1:80",
}, {
name: "https",
proto: "https",
ma: "/dns/foo.com/tcp/443/https",
expected: "https://foo.com:443",
}, {
name: "unknown protocol",
proto: "fancynewproto",
ma: "/dns/foo.com/tcp/80/http",
expected: "",
}}

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
ma, err := multiaddr.NewMultiaddr(tc.ma)
require.NoError(t, err)
res := multiaddrToNative(tc.proto, ma)
require.Equal(t, tc.expected, res)
})
}
}
5 changes: 3 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ require (
github.com/filecoin-project/go-fil-commp-hashhash v0.1.0
github.com/filecoin-project/go-fil-markets v1.23.2
github.com/filecoin-project/go-jsonrpc v0.1.5
github.com/filecoin-project/go-legs v0.4.9
github.com/filecoin-project/go-padreader v0.0.1
github.com/filecoin-project/go-paramfetch v0.0.4
github.com/filecoin-project/go-state-types v0.1.10
Expand Down Expand Up @@ -64,7 +65,7 @@ require (
github.com/ipfs/go-unixfs v0.3.1
github.com/ipld/go-car v0.4.1-0.20220707083113-89de8134e58e
github.com/ipld/go-car/v2 v2.4.2-0.20220707083113-89de8134e58e
github.com/ipld/go-ipld-prime v0.17.0
github.com/ipld/go-ipld-prime v0.18.0
github.com/ipld/go-ipld-selector-text-lite v0.0.1
github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c
github.com/jpillora/backoff v1.0.0
Expand All @@ -83,7 +84,7 @@ require (
github.com/mitchellh/go-homedir v1.1.0
github.com/multiformats/go-multiaddr v0.5.0
github.com/multiformats/go-multibase v0.0.3
github.com/multiformats/go-multihash v0.1.0
github.com/multiformats/go-multihash v0.2.0
github.com/multiformats/go-varint v0.0.6
github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333
github.com/pressly/goose/v3 v3.5.3
Expand Down
12 changes: 8 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -398,8 +398,9 @@ github.com/filecoin-project/go-hamt-ipld/v3 v3.1.0/go.mod h1:bxmzgT8tmeVQA1/gvBw
github.com/filecoin-project/go-indexer-core v0.2.16/go.mod h1:5kCKyhtT9k1vephr9l9SFGX8B/HowXIvOhGCkmbxwbY=
github.com/filecoin-project/go-jsonrpc v0.1.5 h1:ckxqZ09ivBAVf5CSmxxrqqNHC7PJm3GYGtYKiNQ+vGk=
github.com/filecoin-project/go-jsonrpc v0.1.5/go.mod h1:XBBpuKIMaXIIzeqzO1iucq4GvbF8CxmXRFoezRh+Cx4=
github.com/filecoin-project/go-legs v0.4.4 h1:mpMmAOOnamaz0CV9rgeKhEWA8j9kMC+f+UGCGrxKaZo=
github.com/filecoin-project/go-legs v0.4.4/go.mod h1:JQ3hA6xpJdbR8euZ2rO0jkxaMxeidXf0LDnVuqPAe9s=
github.com/filecoin-project/go-legs v0.4.9 h1:9ccbv5zDPqMviEpSpf0TdfKKI64TMYGSiuY2A1EXHFY=
github.com/filecoin-project/go-legs v0.4.9/go.mod h1:JQ3hA6xpJdbR8euZ2rO0jkxaMxeidXf0LDnVuqPAe9s=
github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20/go.mod h1:mPn+LRRd5gEKNAtc+r3ScpW2JRU/pj4NBKdADYWHiak=
github.com/filecoin-project/go-padreader v0.0.1 h1:8h2tVy5HpoNbr2gBRr+WD6zV6VD6XHig+ynSGJg8ZOs=
github.com/filecoin-project/go-padreader v0.0.1/go.mod h1:VYVPJqwpsfmtoHnAmPx6MUwmrK6HIcDqZJiuZhtmfLQ=
Expand Down Expand Up @@ -1022,8 +1023,9 @@ github.com/ipld/go-ipld-prime v0.14.1/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704n
github.com/ipld/go-ipld-prime v0.14.2/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0=
github.com/ipld/go-ipld-prime v0.14.4-0.20211217152141-008fd70fc96f/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0=
github.com/ipld/go-ipld-prime v0.16.0/go.mod h1:axSCuOCBPqrH+gvXr2w9uAOulJqBPhHPT2PjoiiU1qA=
github.com/ipld/go-ipld-prime v0.17.0 h1:+U2peiA3aQsE7mrXjD2nYZaZrCcakoz2Wge8K42Ld8g=
github.com/ipld/go-ipld-prime v0.17.0/go.mod h1:aYcKm5TIvGfY8P3QBKz/2gKcLxzJ1zDaD+o0bOowhgs=
github.com/ipld/go-ipld-prime v0.18.0 h1:xUk7NUBSWHEXdjiOu2sLXouFJOMs0yoYzeI5RAqhYQo=
github.com/ipld/go-ipld-prime v0.18.0/go.mod h1:735yXW548CKrLwVCYXzqx90p5deRJMVVxM9eJ4Qe+qE=
github.com/ipld/go-ipld-prime-proto v0.0.0-20191113031812-e32bd156a1e5/go.mod h1:gcvzoEDBjwycpXt3LBE061wT9f46szXGHAmj9uoP6fU=
github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73 h1:TsyATB2ZRRQGTwafJdgEUQkmjOExRV0DNokcihZxbnQ=
github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73/go.mod h1:2PJ0JgxyB08t0b2WKrcuqI3di0V+5n6RS/LTUJhkoxY=
Expand Down Expand Up @@ -1736,8 +1738,9 @@ github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUj
github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg=
github.com/multiformats/go-multihash v0.0.16/go.mod h1:zhfEIgVnB/rPMfxgFw15ZmGoNaKyNUIE4IWHG/kC+Ag=
github.com/multiformats/go-multihash v0.1.0 h1:CgAgwqk3//SVEw3T+6DqI4mWMyRuDwZtOWcJT0q9+EA=
github.com/multiformats/go-multihash v0.1.0/go.mod h1:RJlXsxt6vHGaia+S8We0ErjhojtKzPP2AH4+kYM7k84=
github.com/multiformats/go-multihash v0.2.0 h1:oytJb9ZA1OUW0r0f9ea18GiaPOo4SXyc7p2movyUuo4=
github.com/multiformats/go-multihash v0.2.0/go.mod h1:WxoMcYG85AZVQUyRyo9s4wULvW5qrI9vb2Lt6evduFc=
github.com/multiformats/go-multistream v0.0.1/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=
github.com/multiformats/go-multistream v0.0.4/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=
github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=
Expand Down Expand Up @@ -2296,8 +2299,9 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 h1:kUhD7nTDoI3fVd9G4ORWrbV5NY0liEs/Jg2pv5f+bBA=
golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM=
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20181106170214-d68db9428509/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
Expand Down
4 changes: 4 additions & 0 deletions node/builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"github.com/filecoin-project/boost/node/impl/common"
"github.com/filecoin-project/boost/node/modules"
"github.com/filecoin-project/boost/node/modules/dtypes"
"github.com/filecoin-project/boost/retrievalmarket/lp2pimpl"
"github.com/filecoin-project/boost/sealingpipeline"
"github.com/filecoin-project/boost/storagemanager"
"github.com/filecoin-project/boost/storagemarket"
Expand Down Expand Up @@ -137,6 +138,7 @@ const (
HandleMigrateProviderFundsKey
HandleDealsKey
HandleRetrievalKey
HandleRetrievalTransportsKey
RunSectorServiceKey

// boost should be started after legacy markets (HandleDealsKey)
Expand Down Expand Up @@ -514,6 +516,8 @@ func ConfigBoost(cfg *config.Boost) Option {
Override(new(rmnet.RetrievalMarketNetwork), lotus_modules.RetrievalNetwork),
Override(new(retrievalmarket.RetrievalProvider), lotus_modules.RetrievalProvider),
Override(HandleRetrievalKey, lotus_modules.HandleRetrieval),
Override(new(*lp2pimpl.TransportsListener), modules.NewTransportsListener(cfg)),
Override(HandleRetrievalTransportsKey, modules.HandleRetrievalTransports),
Override(new(idxprov.MeshCreator), idxprov.NewMeshCreator),
Override(new(provider.Interface), modules.IndexProvider(cfg.IndexProvider)),

Expand Down
7 changes: 7 additions & 0 deletions node/config/doc_gen.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions node/config/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,10 @@ type DealmakingConfig struct {
MaxTransferDuration Duration
// Whether to do commp on the Boost node (local) or on the Sealer (remote)
RemoteCommp bool

// The public multi-address for retrieving deals with booster-http.
// Note: Must be in multiaddr format, eg /dns/foo.com/tcp/443/https
HTTPRetrievalMultiaddr string
}

type FeeConfig struct {
Expand Down
60 changes: 60 additions & 0 deletions node/modules/retrieval.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
package modules

import (
"context"
"fmt"

"github.com/filecoin-project/boost/node/config"
"github.com/filecoin-project/boost/retrievalmarket/lp2pimpl"
"github.com/filecoin-project/boost/retrievalmarket/types"
"github.com/libp2p/go-libp2p-core/host"
"github.com/multiformats/go-multiaddr"
"go.uber.org/fx"
)

func NewTransportsListener(cfg *config.Boost) func(h host.Host) (*lp2pimpl.TransportsListener, error) {
return func(h host.Host) (*lp2pimpl.TransportsListener, error) {
protos := []types.Protocol{}

// Get the libp2p addresses from the Host
if len(h.Addrs()) > 0 {
protos = append(protos, types.Protocol{
Name: "libp2p",
Addresses: h.Addrs(),
})
}

// If there's an http retrieval address specified, add HTTP to the list
// of supported protocols
if cfg.Dealmaking.HTTPRetrievalMultiaddr != "" {
maddr, err := multiaddr.NewMultiaddr(cfg.Dealmaking.HTTPRetrievalMultiaddr)
if err != nil {
msg := "HTTPRetrievalURL must be in multi-address format. "
msg += "Could not parse '%s' as multiaddr: %w"
return nil, fmt.Errorf(msg, cfg.Dealmaking.HTTPRetrievalMultiaddr, err)
}

protos = append(protos, types.Protocol{
Name: "http",
Addresses: []multiaddr.Multiaddr{maddr},
})
}

return lp2pimpl.NewTransportsListener(h, protos), nil
}
}

func HandleRetrievalTransports(lc fx.Lifecycle, l *lp2pimpl.TransportsListener) {
lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error {
log.Debug("starting retrieval transports listener")
l.Start()
return nil
},
OnStop: func(context.Context) error {
log.Debug("stopping retrieval transports listener")
l.Stop()
return nil
},
})
}
Loading

0 comments on commit edfaf9f

Please sign in to comment.