Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

swarm: implement Happy Eyeballs ranking #2365

Merged
merged 1 commit into from
Jun 28, 2023
+148 −92
Merged

swarm: implement Happy Eyeballs ranking #2365

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
141 changes: 85 additions & 56 deletions p2p/net/swarm/dial_ranker.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,11 @@ import (

// The 250ms value is from happy eyeballs RFC 8305. This is a rough estimate of 1 RTT
const (
// duration by which TCP dials are delayed relative to QUIC dial
// duration by which TCP dials are delayed relative to the last QUIC dial
PublicTCPDelay = 250 * time.Millisecond
PrivateTCPDelay = 30 * time.Millisecond

// duration by which QUIC dials are delayed relative to first QUIC dial
// duration by which QUIC dials are delayed relative to previous QUIC dial
PublicQUICDelay = 250 * time.Millisecond
PrivateQUICDelay = 30 * time.Millisecond

Expand All @@ -31,44 +31,42 @@ func NoDelayDialRanker(addrs []ma.Multiaddr) []network.AddrDelay {

// DefaultDialRanker determines the ranking of outgoing connection attempts.
//
// Addresses are grouped into four distinct groups:
// Addresses are grouped into three distinct groups:
//
// - private addresses (localhost and local networks (RFC 1918))
// - public IPv4 addresses
// - public IPv6 addresses
// - public addresses
// - relay addresses
//
// Within each group, the addresses are ranked according to the ranking logic described below.
// We then dial addresses according to this ranking, with short timeouts applied between dial attempts.
// This ranking logic dramatically reduces the number of simultaneous dial attempts, while introducing
// no additional latency in the vast majority of cases.
//
// The private, public IPv4 and public IPv6 groups are dialed in parallel.
// Private and public address groups are dialed in parallel.
// Dialing relay addresses is delayed by 500 ms, if we have any non-relay alternatives.
//
// In a future iteration, IPv6 will be given a headstart over IPv4, as recommended by Happy Eyeballs RFC 8305.
// This is not enabled yet, since some ISPs are still IPv4-only, and dialing IPv6 addresses will therefore
// always fail.
// The correct solution is to detect this situation, and not attempt to dial IPv6 addresses at all.
// IPv6 blackhole detection is tracked in https://github.com/libp2p/go-libp2p/issues/1605.
// Within each group (private, public, relay addresses) we apply the following ranking logic:
//
// Within each group (private, public IPv4, public IPv6, relay addresses) we apply the following
// ranking logic:
//
// 1. If two QUIC addresses are present, dial the QUIC address with the lowest port first:
// This is more likely to be the listen port. After this we dial the rest of the QUIC addresses delayed by
// 250ms (PublicQUICDelay) for public addresses, and 30ms (PrivateQUICDelay) for local addresses.
// 2. If a QUIC or WebTransport address is present, TCP addresses dials are delayed relative to the last QUIC dial:
// 1. If both IPv6 QUIC and IPv4 QUIC addresses are present, we do a Happy Eyeballs RFC 8305 style ranking.
// First dial the IPv6 QUIC address with the lowest port. After this we dial the IPv4 QUIC address with
// the lowest port delayed by 250ms (PublicQUICDelay) for public addresses, and 30ms (PrivateQUICDelay)
// for local addresses. After this we dial all the rest of the addresses delayed by 250ms (PublicQUICDelay)
// for public addresses, and 30ms (PrivateQUICDelay) for local addresses.
// 2. If only one of QUIC IPv6 or QUIC IPv4 addresses are present, dial the QUIC address with the lowest port
// first. After this we dial the rest of the QUIC addresses delayed by 250ms (PublicQUICDelay) for public
// addresses, and 30ms (PrivateQUICDelay) for local addresses.
// 3. If a QUIC or WebTransport address is present, TCP addresses dials are delayed relative to the last QUIC dial:
// We prefer to end up with a QUIC connection. For public addresses, the delay introduced is 250ms (PublicTCPDelay),
// and for private addresses 30ms (PrivateTCPDelay).
//
// We dial lowest ports first for QUIC addresses as they are more likely to be the listen port.
func DefaultDialRanker(addrs []ma.Multiaddr) []network.AddrDelay {
relay, addrs := filterAddrs(addrs, isRelayAddr)
pvt, addrs := filterAddrs(addrs, manet.IsPrivateAddr)
ip4, addrs := filterAddrs(addrs, func(a ma.Multiaddr) bool { return isProtocolAddr(a, ma.P_IP4) })
ip6, addrs := filterAddrs(addrs, func(a ma.Multiaddr) bool { return isProtocolAddr(a, ma.P_IP6) })
public, addrs := filterAddrs(addrs, func(a ma.Multiaddr) bool { return isProtocolAddr(a, ma.P_IP4) || isProtocolAddr(a, ma.P_IP6) })

var relayOffset time.Duration = 0
if len(ip4) > 0 || len(ip6) > 0 {
var relayOffset time.Duration
if len(public) > 0 {
// if there is a public direct address available delay relay dials
relayOffset = RelayDelay
}
Expand All @@ -77,70 +75,97 @@ func DefaultDialRanker(addrs []ma.Multiaddr) []network.AddrDelay {
for i := 0; i < len(addrs); i++ {
res = append(res, network.AddrDelay{Addr: addrs[i], Delay: 0})
}

res = append(res, getAddrDelay(pvt, PrivateTCPDelay, PrivateQUICDelay, 0)...)
res = append(res, getAddrDelay(ip4, PublicTCPDelay, PublicQUICDelay, 0)...)
res = append(res, getAddrDelay(ip6, PublicTCPDelay, PublicQUICDelay, 0)...)
res = append(res, getAddrDelay(public, PublicTCPDelay, PublicQUICDelay, 0)...)
res = append(res, getAddrDelay(relay, PublicTCPDelay, PublicQUICDelay, relayOffset)...)
return res
}

// getAddrDelay ranks a group of addresses(private, ip4, ip6) according to the ranking logic
// explained in defaultDialRanker.
// getAddrDelay ranks a group of addresses according to the ranking logic explained in
// documentation for defaultDialRanker.
// offset is used to delay all addresses by a fixed duration. This is useful for delaying all relay
// addresses relative to direct addresses
// addresses relative to direct addresses.
func getAddrDelay(addrs []ma.Multiaddr, tcpDelay time.Duration, quicDelay time.Duration,
offset time.Duration) []network.AddrDelay {

sort.Slice(addrs, func(i, j int) bool { return score(addrs[i]) < score(addrs[j]) })

// If the first address is (QUIC, IPv6), make the second address (QUIC, IPv4).
marten-seemann marked this conversation as resolved.
Show resolved Hide resolved
happyEyeballs := false
if len(addrs) > 0 {
if isQUICAddr(addrs[0]) && isProtocolAddr(addrs[0], ma.P_IP6) {
for i := 1; i < len(addrs); i++ {
if isQUICAddr(addrs[i]) && isProtocolAddr(addrs[i], ma.P_IP4) {
// make IPv4 address the second element
if i > 1 {
a := addrs[i]
copy(addrs[2:], addrs[1:i])
addrs[1] = a
}
happyEyeballs = true
break
}
}
}
}

res := make([]network.AddrDelay, 0, len(addrs))
quicCount := 0
for _, a := range addrs {
delay := offset

var totalTCPDelay time.Duration
for i, addr := range addrs {
var delay time.Duration
switch {
case isProtocolAddr(a, ma.P_QUIC) || isProtocolAddr(a, ma.P_QUIC_V1):
// For QUIC addresses we dial a single address first and then wait for QUICDelay
// After QUICDelay we dial rest of the QUIC addresses
if quicCount > 0 {
delay += quicDelay
case isQUICAddr(addr):
// For QUIC addresses we dial an IPv6 address, then after quicDelay an IPv4
// address, then after quicDelay we dial rest of the addresses.
if i == 1 {
delay = quicDelay
}
quicCount++
case isProtocolAddr(a, ma.P_TCP):
if quicCount >= 2 {
delay += 2 * quicDelay
} else if quicCount == 1 {
delay += tcpDelay
if i > 1 && happyEyeballs {
delay = 2 * quicDelay
} else if i > 1 {
delay = quicDelay
}
totalTCPDelay = delay + tcpDelay
case isProtocolAddr(addr, ma.P_TCP):
delay = totalTCPDelay
}
res = append(res, network.AddrDelay{Addr: a, Delay: delay})
res = append(res, network.AddrDelay{Addr: addr, Delay: offset + delay})
}
return res
}

// score scores a multiaddress for dialing delay. lower is better
// score scores a multiaddress for dialing delay. Lower is better.
// The lower 16 bits of the result are the port. Low ports are ranked higher because they're
// more likely to be listen addresses.
// The addresses are ranked as:
// QUICv1 IPv6 > QUICdraft29 IPv6 > QUICv1 IPv4 > QUICdraft29 IPv4 >
// WebTransport IPv6 > WebTransport IPv4 > TCP IPv6 > TCP IPv4
func score(a ma.Multiaddr) int {
// the lower 16 bits of the result are the relavant port
// the higher bits rank the protocol
// low ports are ranked higher because they're more likely to
// be listen addresses
ip4Weight := 0
if isProtocolAddr(a, ma.P_IP4) {
ip4Weight = 1 << 18
}

if _, err := a.ValueForProtocol(ma.P_WEBTRANSPORT); err == nil {
p, _ := a.ValueForProtocol(ma.P_UDP)
pi, _ := strconv.Atoi(p) // cannot error
return pi + (1 << 18)
pi, _ := strconv.Atoi(p)
return ip4Weight + (1 << 19) + pi
}
if _, err := a.ValueForProtocol(ma.P_QUIC); err == nil {
p, _ := a.ValueForProtocol(ma.P_UDP)
pi, _ := strconv.Atoi(p) // cannot error
return pi + (1 << 17)
pi, _ := strconv.Atoi(p)
return ip4Weight + pi + (1 << 17)
}
if _, err := a.ValueForProtocol(ma.P_QUIC_V1); err == nil {
p, _ := a.ValueForProtocol(ma.P_UDP)
pi, _ := strconv.Atoi(p) // cannot error
return pi
pi, _ := strconv.Atoi(p)
return ip4Weight + pi
}

if p, err := a.ValueForProtocol(ma.P_TCP); err == nil {
pi, _ := strconv.Atoi(p) // cannot error
return pi + (1 << 19)
pi, _ := strconv.Atoi(p)
return ip4Weight + pi + (1 << 20)
}
return (1 << 30)
}
Expand All @@ -157,6 +182,10 @@ func isProtocolAddr(a ma.Multiaddr, p int) bool {
return found
}

func isQUICAddr(a ma.Multiaddr) bool {
return isProtocolAddr(a, ma.P_QUIC) || isProtocolAddr(a, ma.P_QUIC_V1)
}

// filterAddrs filters an address slice in place
func filterAddrs(addrs []ma.Multiaddr, f func(a ma.Multiaddr) bool) (filtered, rest []ma.Multiaddr) {
j := 0
Expand Down
99 changes: 63 additions & 36 deletions p2p/net/swarm/dial_ranker_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ func sortAddrDelays(addrDelays []network.AddrDelay) {
})
}

func TestNoDelayRanker(t *testing.T) {
func TestNoDelayDialRanker(t *testing.T) {
q1 := ma.StringCast("/ip4/1.2.3.4/udp/1/quic")
q1v1 := ma.StringCast("/ip4/1.2.3.4/udp/1/quic-v1")
wt1 := ma.StringCast("/ip4/1.2.3.4/udp/1/quic-v1/webtransport/")
Expand All @@ -36,7 +36,7 @@ func TestNoDelayRanker(t *testing.T) {
output []network.AddrDelay
}{
{
name: "quic-ranking",
name: "quic+webtransport filtered when quicv1",
addrs: []ma.Multiaddr{q1, q2, q3, q4, q1v1, q2v1, q3v1, wt1, t1},
output: []network.AddrDelay{
{Addr: q1, Delay: 0},
Expand Down Expand Up @@ -89,7 +89,7 @@ func TestDelayRankerQUICDelay(t *testing.T) {
output []network.AddrDelay
}{
{
name: "single quic dialed first",
name: "quic-ipv4",
addrs: []ma.Multiaddr{q1, q2, q3, q4},
output: []network.AddrDelay{
{Addr: q1, Delay: 0},
Expand All @@ -99,37 +99,49 @@ func TestDelayRankerQUICDelay(t *testing.T) {
},
},
{
name: "quicv1 dialed before quic",
addrs: []ma.Multiaddr{q1, q2v1, q3, q4},
name: "quic-ipv6",
addrs: []ma.Multiaddr{q1v16, q2v16, q3v16},
output: []network.AddrDelay{
{Addr: q2v1, Delay: 0},
{Addr: q1, Delay: PublicQUICDelay},
{Addr: q3, Delay: PublicQUICDelay},
{Addr: q4, Delay: PublicQUICDelay},
{Addr: q1v16, Delay: 0},
{Addr: q2v16, Delay: PublicQUICDelay},
{Addr: q3v16, Delay: PublicQUICDelay},
},
},
{
name: "quic-quic-v1-webtransport",
addrs: []ma.Multiaddr{q1, q2, q3, q4, q1v1, q2v1, q3v1, wt1},
name: "quic-ip4-ip6",
addrs: []ma.Multiaddr{q1, q1v16, q2v1, q3, q4},
output: []network.AddrDelay{
{Addr: q1v1, Delay: 0},
{Addr: q1, Delay: PublicQUICDelay},
{Addr: q2, Delay: PublicQUICDelay},
{Addr: q3, Delay: PublicQUICDelay},
{Addr: q4, Delay: PublicQUICDelay},
{Addr: q1v16, Delay: 0},
{Addr: q2v1, Delay: PublicQUICDelay},
{Addr: q3v1, Delay: PublicQUICDelay},
{Addr: wt1, Delay: PublicQUICDelay},
{Addr: q1, Delay: 2 * PublicQUICDelay},
{Addr: q3, Delay: 2 * PublicQUICDelay},
{Addr: q4, Delay: 2 * PublicQUICDelay},
},
},
{
name: "ipv6",
addrs: []ma.Multiaddr{q1v16, q2v16, q3v16, q1},
name: "quic-quic-v1-webtransport",
addrs: []ma.Multiaddr{q1v16, q1, q2, q3, q4, q1v1, q2v1, q3v1, wt1},
output: []network.AddrDelay{
{Addr: q1, Delay: 0},
{Addr: q1v16, Delay: 0},
{Addr: q2v16, Delay: PublicQUICDelay},
{Addr: q3v16, Delay: PublicQUICDelay},
{Addr: q1v1, Delay: PublicQUICDelay},
{Addr: q2v1, Delay: 2 * PublicQUICDelay},
{Addr: q3v1, Delay: 2 * PublicQUICDelay},
{Addr: q1, Delay: 2 * PublicQUICDelay},
{Addr: q2, Delay: 2 * PublicQUICDelay},
{Addr: q3, Delay: 2 * PublicQUICDelay},
{Addr: q4, Delay: 2 * PublicQUICDelay},
{Addr: wt1, Delay: 2 * PublicQUICDelay},
},
},
{
name: "wt-ranking",
addrs: []ma.Multiaddr{q1v16, q2v16, q3v16, q2, wt1},
output: []network.AddrDelay{
{Addr: q1v16, Delay: 0},
{Addr: q2, Delay: PublicQUICDelay},
{Addr: wt1, Delay: 2 * PublicQUICDelay},
{Addr: q2v16, Delay: 2 * PublicQUICDelay},
{Addr: q3v16, Delay: 2 * PublicQUICDelay},
},
},
}
Expand All @@ -152,11 +164,18 @@ func TestDelayRankerQUICDelay(t *testing.T) {
}

func TestDelayRankerTCPDelay(t *testing.T) {

q1 := ma.StringCast("/ip4/1.2.3.4/udp/1/quic")
q1v1 := ma.StringCast("/ip4/1.2.3.4/udp/1/quic-v1")
q2 := ma.StringCast("/ip4/1.2.3.4/udp/2/quic")
q2v1 := ma.StringCast("/ip4/1.2.3.4/udp/2/quic-v1")
q3 := ma.StringCast("/ip4/1.2.3.4/udp/3/quic")

q1v16 := ma.StringCast("/ip6/1::2/udp/1/quic-v1")
q2v16 := ma.StringCast("/ip6/1::2/udp/2/quic-v1")
q3v16 := ma.StringCast("/ip6/1::2/udp/3/quic-v1")

t1 := ma.StringCast("/ip4/1.2.3.5/tcp/1/")
t1v6 := ma.StringCast("/ip6/1::2/tcp/1")
t2 := ma.StringCast("/ip4/1.2.3.4/tcp/2")

testCase := []struct {
Expand All @@ -165,28 +184,36 @@ func TestDelayRankerTCPDelay(t *testing.T) {
output []network.AddrDelay
}{
{
name: "2 quic with tcp",
addrs: []ma.Multiaddr{q1, q2v1, t1, t2},
name: "quic-with-tcp-ip6-ip4",
addrs: []ma.Multiaddr{q1, q1v1, q1v16, q2v16, q3v16, q2v1, t1, t2},
output: []network.AddrDelay{
{Addr: q2v1, Delay: 0},
{Addr: q1, Delay: PublicQUICDelay},
{Addr: t1, Delay: PublicQUICDelay + PublicTCPDelay},
{Addr: t2, Delay: PublicQUICDelay + PublicTCPDelay},
{Addr: q1v16, Delay: 0},
{Addr: q1v1, Delay: PublicQUICDelay},
{Addr: q1, Delay: 2 * PublicQUICDelay},
{Addr: q2v16, Delay: 2 * PublicQUICDelay},
{Addr: q3v16, Delay: 2 * PublicQUICDelay},
{Addr: q2v1, Delay: 2 * PublicQUICDelay},
{Addr: t1, Delay: 3 * PublicQUICDelay},
{Addr: t2, Delay: 3 * PublicQUICDelay},
},
},
{
name: "1 quic with tcp",
addrs: []ma.Multiaddr{q1, t1, t2},
name: "quic-ip4-with-tcp",
addrs: []ma.Multiaddr{q1, q2, q3, t1, t2, t1v6},
output: []network.AddrDelay{
{Addr: q1, Delay: 0},
{Addr: t1, Delay: PublicTCPDelay},
{Addr: t2, Delay: PublicTCPDelay},
{Addr: q2, Delay: PublicQUICDelay},
{Addr: q3, Delay: PublicQUICDelay},
{Addr: t1, Delay: PublicQUICDelay + PublicTCPDelay},
{Addr: t2, Delay: PublicQUICDelay + PublicTCPDelay},
{Addr: t1v6, Delay: PublicQUICDelay + PublicTCPDelay},
},
},
{
name: "no quic",
addrs: []ma.Multiaddr{t1, t2},
name: "tcp-ip4-ip6",
addrs: []ma.Multiaddr{t1, t2, t1v6},
output: []network.AddrDelay{
{Addr: t1v6, Delay: 0},
{Addr: t1, Delay: 0},
{Addr: t2, Delay: 0},
},
Expand Down