Skip to content

Commit

Permalink
fix: disable provide over HTTP with Routing.Type=auto (#9511)
Browse files Browse the repository at this point in the history
Closes #9504
  • Loading branch information
ajnavarro authored Jan 3, 2023
1 parent ae774a3 commit d6069b9
Show file tree
Hide file tree
Showing 8 changed files with 111 additions and 13 deletions.
20 changes: 19 additions & 1 deletion core/node/libp2p/routingopt.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ package libp2p

import (
"context"
"os"
"strings"
"time"

"github.com/ipfs/go-datastore"
Expand Down Expand Up @@ -30,6 +32,13 @@ var defaultHTTPRouters = []string{
// TODO: add an independent router from Cloudflare
}

func init() {
// Override HTTP routers if custom ones were passed via env
if routers := os.Getenv("IPFS_HTTP_ROUTERS"); routers != "" {
defaultHTTPRouters = strings.Split(routers, " ")
}
}

// ConstructDefaultRouting returns routers used when Routing.Type is unset or set to "auto"
func ConstructDefaultRouting(peerID string, addrs []string, privKey string) func(
ctx context.Context,
Expand Down Expand Up @@ -67,8 +76,17 @@ func ConstructDefaultRouting(peerID string, addrs []string, privKey string) func
if err != nil {
return nil, err
}

r := &irouting.Composer{
GetValueRouter: routinghelpers.Null{},
PutValueRouter: routinghelpers.Null{},
ProvideRouter: routinghelpers.Null{}, // modify this when indexers supports provide
FindPeersRouter: routinghelpers.Null{},
FindProvidersRouter: httpRouter,
}

routers = append(routers, &routinghelpers.ParallelRouter{
Router: httpRouter,
Router: r,
IgnoreError: true, // https://github.com/ipfs/kubo/pull/9475#discussion_r1042507387
Timeout: 15 * time.Second, // 5x server value from https://github.com/ipfs/kubo/pull/9475#discussion_r1042428529
ExecuteAfter: 0,
Expand Down
17 changes: 17 additions & 0 deletions docs/environment-variables.md
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,23 @@ $ ipfs resolve -r /ipns/dnslink-test2.example.com
/ipfs/bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am
```

## `IPFS_HTTP_ROUTERS`

Overrides all implicit HTTP routers enabled when `Routing.Type=auto` with
the space-separated list of URLs provided in this variable.
Useful for testing and debugging in offline contexts.

Example:

```console
$ ipfs config Routing.Type auto
$ IPFS_HTTP_ROUTERS="http://127.0.0.1:7423" ipfs daemon
```

The above will replace implicit HTTP routers with single one, allowing for
inspection/debug of HTTP requests sent by Kubo via `while true ; do nc -l 7423; done`
or more advanced tools like [mitmproxy](https://docs.mitmproxy.org/stable/#mitmproxy).

## `LIBP2P_TCP_REUSEPORT`

Kubo tries to reuse the same source port for all connections to improve NAT
Expand Down
4 changes: 2 additions & 2 deletions docs/examples/kubo-as-a-library/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ require (
github.com/libp2p/go-libp2p-pubsub v0.8.2 // indirect
github.com/libp2p/go-libp2p-pubsub-router v0.6.0 // indirect
github.com/libp2p/go-libp2p-record v0.2.0 // indirect
github.com/libp2p/go-libp2p-routing-helpers v0.4.1 // indirect
github.com/libp2p/go-libp2p-routing-helpers v0.5.0 // indirect
github.com/libp2p/go-libp2p-xor v0.1.0 // indirect
github.com/libp2p/go-mplex v0.7.0 // indirect
github.com/libp2p/go-msgio v0.2.0 // indirect
Expand Down Expand Up @@ -198,7 +198,7 @@ require (
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/dig v1.15.0 // indirect
go.uber.org/fx v1.18.2 // indirect
go.uber.org/multierr v1.8.0 // indirect
go.uber.org/multierr v1.9.0 // indirect
go.uber.org/zap v1.24.0 // indirect
go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
golang.org/x/crypto v0.3.0 // indirect
Expand Down
8 changes: 4 additions & 4 deletions docs/examples/kubo-as-a-library/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -793,8 +793,8 @@ github.com/libp2p/go-libp2p-quic-transport v0.10.0/go.mod h1:RfJbZ8IqXIhxBRm5hqU
github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q=
github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0=
github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk=
github.com/libp2p/go-libp2p-routing-helpers v0.4.1 h1:rOlZiFpUt7SgHm4w62MBvWaQ4UHh7bVJnSnor6RN7j8=
github.com/libp2p/go-libp2p-routing-helpers v0.4.1/go.mod h1:dYEAgkVhqho3/YKxfOEGdFMIcWfAFNlZX8iAIihYA2E=
github.com/libp2p/go-libp2p-routing-helpers v0.5.0 h1:Byujua1X9MeTzbF54i5OwjUNopeg7PYBykuNow/w3p4=
github.com/libp2p/go-libp2p-routing-helpers v0.5.0/go.mod h1:wwK/XSLt6njjO7sRbjhf8w7PGBOfdntMQ2mOQPZ5s/Q=
github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8=
github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g=
github.com/libp2p/go-libp2p-secio v0.2.1/go.mod h1:cWtZpILJqkqrSkiYcDBh5lA3wbT2Q+hz3rJQq3iftD8=
Expand Down Expand Up @@ -1377,8 +1377,8 @@ go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+
go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8=
go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
Expand Down
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ require (
github.com/libp2p/go-libp2p-pubsub v0.8.2
github.com/libp2p/go-libp2p-pubsub-router v0.6.0
github.com/libp2p/go-libp2p-record v0.2.0
github.com/libp2p/go-libp2p-routing-helpers v0.4.0
github.com/libp2p/go-libp2p-routing-helpers v0.5.0
github.com/libp2p/go-libp2p-testing v0.12.0
github.com/libp2p/go-socket-activation v0.1.0
github.com/miekg/dns v1.1.50
Expand Down Expand Up @@ -231,7 +231,7 @@ require (
go.opentelemetry.io/otel/metric v0.30.0 // indirect
go.opentelemetry.io/proto/otlp v0.16.0 // indirect
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/multierr v1.8.0 // indirect
go.uber.org/multierr v1.9.0 // indirect
go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
golang.org/x/exp v0.0.0-20221205204356-47842c84f3db // indirect
golang.org/x/net v0.3.0 // indirect
Expand Down
8 changes: 4 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -828,8 +828,8 @@ github.com/libp2p/go-libp2p-quic-transport v0.10.0/go.mod h1:RfJbZ8IqXIhxBRm5hqU
github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q=
github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0=
github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk=
github.com/libp2p/go-libp2p-routing-helpers v0.4.0 h1:b7y4aixQ7AwbqYfcOQ6wTw8DQvuRZeTAA0Od3YYN5yc=
github.com/libp2p/go-libp2p-routing-helpers v0.4.0/go.mod h1:dYEAgkVhqho3/YKxfOEGdFMIcWfAFNlZX8iAIihYA2E=
github.com/libp2p/go-libp2p-routing-helpers v0.5.0 h1:Byujua1X9MeTzbF54i5OwjUNopeg7PYBykuNow/w3p4=
github.com/libp2p/go-libp2p-routing-helpers v0.5.0/go.mod h1:wwK/XSLt6njjO7sRbjhf8w7PGBOfdntMQ2mOQPZ5s/Q=
github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8=
github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g=
github.com/libp2p/go-libp2p-secio v0.2.1/go.mod h1:cWtZpILJqkqrSkiYcDBh5lA3wbT2Q+hz3rJQq3iftD8=
Expand Down Expand Up @@ -1441,8 +1441,8 @@ go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+
go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8=
go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
Expand Down
9 changes: 9 additions & 0 deletions routing/composer.go
Original file line number Diff line number Diff line change
Expand Up @@ -100,9 +100,18 @@ func (c *Composer) GetValue(ctx context.Context, key string, opts ...routing.Opt
func (c *Composer) SearchValue(ctx context.Context, key string, opts ...routing.Option) (<-chan []byte, error) {
log.Debug("composer: calling searchValue: ", key)
ch, err := c.GetValueRouter.SearchValue(ctx, key, opts...)

// avoid nil channels on implementations not supporting SearchValue method.
if err == routing.ErrNotFound && ch == nil {
out := make(chan []byte)
close(out)
return out, err
}

if err != nil {
log.Debug("composer: calling searchValue error: ", key, err)
}

return ch, err
}

Expand Down
54 changes: 54 additions & 0 deletions test/sharness/t0172-content-routing-over-http.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
#!/usr/bin/env bash

test_description="Test content routing over HTTP"

. lib/test-lib.sh


if ! test_have_prereq SOCAT; then
skip_all="skipping '$test_description': socat is not available"
test_done
fi

test_init_ipfs

# Run listener on a free port to log HTTP requests sent by Kubo in Routing.Type=auto mode
export ROUTER_PORT=$(comm -23 <(seq 49152 65535 | sort) <(ss -Htan | awk '{print $4}' | cut -d':' -f2) | head -n 1)
export IPFS_HTTP_ROUTERS="http://127.0.0.1:$ROUTER_PORT"

test_launch_ipfs_daemon

test_expect_success "start HTTP router proxy" '
socat TCP-LISTEN:$ROUTER_PORT,reuseaddr,fork,bind=127.0.0.1 STDOUT > http_requests &
NCPID=$!
'

## HTTP GETs

test_expect_success 'create unique CID without adding it to the local datastore' '
WANT_CID=$(date +"%FT%T.%N%z" | ipfs add -qn)
'

test_expect_success 'expect HTTP request for unknown CID' '
ipfs routing findprovs --timeout 3s "$WANT_CID" &&
test_should_contain "GET /routing/v1/providers/$WANT_CID" http_requests
'

## HTTP PUTs

test_expect_success 'add new CID to the local datastore' '
ADD_CID=$(date +"%FT%T.%N%z" | ipfs add -q)
'

# cid.contact supports GET-only: https://github.com/ipfs/kubo/issues/9504
# which means no announcements over HTTP should be made.
test_expect_success 'expect no HTTP requests to be sent with locally added CID' '
test_should_not_contain "$ADD_CID" http_requests
'

test_expect_success "stop nc" '
kill "$NCPID" && wait "$NCPID" || true
'

test_kill_ipfs_daemon
test_done

0 comments on commit d6069b9

Please sign in to comment.