Skip to content

Commit

Permalink
swarm: implement Happy Eyeballs ranking
Browse files Browse the repository at this point in the history
  • Loading branch information
@sukunrt
sukunrt committed Jun 14, 2023
1 parent 5c1b358 commit 9bf1c22
Show file tree
Hide file tree
Showing 2 changed files with 142 additions and 91 deletions.
134 changes: 79 additions & 55 deletions p2p/net/swarm/dial_ranker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,11 @@ import (

// The 250ms value is from happy eyeballs RFC 8305. This is a rough estimate of 1 RTT
const (
// duration by which TCP dials are delayed relative to QUIC dial
// duration by which TCP dials are delayed relative to the last QUIC dial
PublicTCPDelay = 250 * time.Millisecond
PrivateTCPDelay = 30 * time.Millisecond

// duration by which QUIC dials are delayed relative to first QUIC dial
// duration by which QUIC dials are delayed relative to previous QUIC dial
PublicQUICDelay = 250 * time.Millisecond
PrivateQUICDelay = 30 * time.Millisecond

Expand All @@ -31,44 +31,42 @@ func NoDelayDialRanker(addrs []ma.Multiaddr) []network.AddrDelay {

// DefaultDialRanker determines the ranking of outgoing connection attempts.
//
// Addresses are grouped into four distinct groups:
// Addresses are grouped into three distinct groups:
//
// - private addresses (localhost and local networks (RFC 1918))
// - public IPv4 addresses
// - public IPv6 addresses
// - public addresses
// - relay addresses
//
// Within each group, the addresses are ranked according to the ranking logic described below.
// We then dial addresses according to this ranking, with short timeouts applied between dial attempts.
// This ranking logic dramatically reduces the number of simultaneous dial attempts, while introducing
// no additional latency in the vast majority of cases.
//
// The private, public IPv4 and public IPv6 groups are dialed in parallel.
// Private and public address groups are dialed in parallel.
// Dialing relay addresses is delayed by 500 ms, if we have any non-relay alternatives.
//
// In a future iteration, IPv6 will be given a headstart over IPv4, as recommended by Happy Eyeballs RFC 8305.
// This is not enabled yet, since some ISPs are still IPv4-only, and dialing IPv6 addresses will therefore
// always fail.
// The correct solution is to detect this situation, and not attempt to dial IPv6 addresses at all.
// IPv6 blackhole detection is tracked in https://github.com/libp2p/go-libp2p/issues/1605.
// Within each group (private, public, relay addresses) we apply the following ranking logic:
//
// Within each group (private, public IPv4, public IPv6, relay addresses) we apply the following
// ranking logic:
//
// 1. If two QUIC addresses are present, dial the QUIC address with the lowest port first:
// This is more likely to be the listen port. After this we dial the rest of the QUIC addresses delayed by
// 250ms (PublicQUICDelay) for public addresses, and 30ms (PrivateQUICDelay) for local addresses.
// 2. If a QUIC or WebTransport address is present, TCP addresses dials are delayed relative to the last QUIC dial:
// 1. If both IPv6 QUIC and IPv4 QUIC addresses are present, we do a Happy Eyeballs RFC 8305 style ranking.
// First dial the IPv6 QUIC address with the lowest port. After this we dial the IPv4 QUIC address with
// the lowest port delayed by 250ms (PublicQUICDelay) for public addresses, and 30ms (PrivateQUICDelay)
// for local addresses. After this we dial all the rest of the addresses delayed by 250ms (PublicQUICDelay)
// for public addresses, and 30ms (PrivateQUICDelay) for local addresses.
// 2. If only one of QUIC IPv6 or QUIC IPv4 addresses are present, dial the QUIC address with the lowest port
// first. After this we dial the rest of the QUIC addresses delayed by 250ms (PublicQUICDelay) for public
// addresses, and 30ms (PrivateQUICDelay) for local addresses.
// 3. If a QUIC or WebTransport address is present, TCP addresses dials are delayed relative to the last QUIC dial:
// We prefer to end up with a QUIC connection. For public addresses, the delay introduced is 250ms (PublicTCPDelay),
// and for private addresses 30ms (PrivateTCPDelay).
//
// We dial lowest ports first for QUIC addresses as they are more likely to be the listen port.
func DefaultDialRanker(addrs []ma.Multiaddr) []network.AddrDelay {
relay, addrs := filterAddrs(addrs, isRelayAddr)
pvt, addrs := filterAddrs(addrs, manet.IsPrivateAddr)
ip4, addrs := filterAddrs(addrs, func(a ma.Multiaddr) bool { return isProtocolAddr(a, ma.P_IP4) })
ip6, addrs := filterAddrs(addrs, func(a ma.Multiaddr) bool { return isProtocolAddr(a, ma.P_IP6) })
public, addrs := filterAddrs(addrs, func(a ma.Multiaddr) bool { return isProtocolAddr(a, ma.P_IP4) || isProtocolAddr(a, ma.P_IP6) })

var relayOffset time.Duration = 0
if len(ip4) > 0 || len(ip6) > 0 {
if len(public) > 0 {
// if there is a public direct address available delay relay dials
relayOffset = RelayDelay
}
Expand All @@ -77,70 +75,92 @@ func DefaultDialRanker(addrs []ma.Multiaddr) []network.AddrDelay {
for i := 0; i < len(addrs); i++ {
res = append(res, network.AddrDelay{Addr: addrs[i], Delay: 0})
}

res = append(res, getAddrDelay(pvt, PrivateTCPDelay, PrivateQUICDelay, 0)...)
res = append(res, getAddrDelay(ip4, PublicTCPDelay, PublicQUICDelay, 0)...)
res = append(res, getAddrDelay(ip6, PublicTCPDelay, PublicQUICDelay, 0)...)
res = append(res, getAddrDelay(public, PublicTCPDelay, PublicQUICDelay, 0)...)
res = append(res, getAddrDelay(relay, PublicTCPDelay, PublicQUICDelay, relayOffset)...)
return res
}

// getAddrDelay ranks a group of addresses(private, ip4, ip6) according to the ranking logic
// explained in defaultDialRanker.
// getAddrDelay ranks a group of addresses according to the ranking logic explained in
// documentation for defaultDialRanker.
// offset is used to delay all addresses by a fixed duration. This is useful for delaying all relay
// addresses relative to direct addresses
// addresses relative to direct addresses.
func getAddrDelay(addrs []ma.Multiaddr, tcpDelay time.Duration, quicDelay time.Duration,
offset time.Duration) []network.AddrDelay {

sort.Slice(addrs, func(i, j int) bool { return score(addrs[i]) < score(addrs[j]) })

// If the first address is (QUIC, IPv6), make the second address (QUIC, IPv4).
happyEyeballs := false
if len(addrs) > 0 {
if isQUICAddr(addrs[0]) && isProtocolAddr(addrs[0], ma.P_IP6) {
for i := 1; i < len(addrs); i++ {
if isQUICAddr(addrs[i]) && isProtocolAddr(addrs[i], ma.P_IP4) {
addrs[i], addrs[1] = addrs[1], addrs[i]
happyEyeballs = true
break
}
}
}
}

res := make([]network.AddrDelay, 0, len(addrs))
quicCount := 0
for _, a := range addrs {
delay := offset

var totalTCPDelay time.Duration = 0
for i, addr := range addrs {
var delay time.Duration = 0
switch {
case isProtocolAddr(a, ma.P_QUIC) || isProtocolAddr(a, ma.P_QUIC_V1):
// For QUIC addresses we dial a single address first and then wait for QUICDelay
// After QUICDelay we dial rest of the QUIC addresses
if quicCount > 0 {
delay += quicDelay
case isQUICAddr(addr):
// For QUIC addresses we dial an IPv6 address, then after quicDelay an IPv4
// address, then after quicDelay we dial rest of the addresses.
if i == 1 {
delay = quicDelay
}
quicCount++
case isProtocolAddr(a, ma.P_TCP):
if quicCount >= 2 {
delay += 2 * quicDelay
} else if quicCount == 1 {
delay += tcpDelay
if i > 1 && happyEyeballs {
delay = 2 * quicDelay
} else if i > 1 {
delay = quicDelay
}
totalTCPDelay = delay + tcpDelay
case isProtocolAddr(addr, ma.P_TCP):
delay = totalTCPDelay
}
res = append(res, network.AddrDelay{Addr: a, Delay: delay})
res = append(res, network.AddrDelay{Addr: addr, Delay: offset + delay})
}
return res
}

// score scores a multiaddress for dialing delay. lower is better
// score scores a multiaddress for dialing delay. Lower is better.
// The lower 16 bits of the result are the port. Low ports are ranked higher because they're
// more likely to be listen addresses.
// The addresses are ranked as:
// QUICv1 IPv6 > QUICdraft29 IPv6 > QUICv1 IPv4 > QUICdraft29 IPv4 >
// WebTransport IPv6 > WebTransport IPv4 > TCP IPv6 > TCP IPv4
func score(a ma.Multiaddr) int {
// the lower 16 bits of the result are the relavant port
// the higher bits rank the protocol
// low ports are ranked higher because they're more likely to
// be listen addresses
ip4Weight := 0
if isProtocolAddr(a, ma.P_IP4) {
ip4Weight = (1 << 18)
}

if _, err := a.ValueForProtocol(ma.P_WEBTRANSPORT); err == nil {
p, _ := a.ValueForProtocol(ma.P_UDP)
pi, _ := strconv.Atoi(p) // cannot error
return pi + (1 << 18)
pi, _ := strconv.Atoi(p)
return ip4Weight + (1 << 19) + pi
}
if _, err := a.ValueForProtocol(ma.P_QUIC); err == nil {
p, _ := a.ValueForProtocol(ma.P_UDP)
pi, _ := strconv.Atoi(p) // cannot error
return pi + (1 << 17)
pi, _ := strconv.Atoi(p)
return ip4Weight + pi + (1 << 17)
}
if _, err := a.ValueForProtocol(ma.P_QUIC_V1); err == nil {
p, _ := a.ValueForProtocol(ma.P_UDP)
pi, _ := strconv.Atoi(p) // cannot error
return pi
pi, _ := strconv.Atoi(p)
return ip4Weight + pi
}

if p, err := a.ValueForProtocol(ma.P_TCP); err == nil {
pi, _ := strconv.Atoi(p) // cannot error
return pi + (1 << 19)
pi, _ := strconv.Atoi(p)
return ip4Weight + pi + (1 << 20)
}
return (1 << 30)
}
Expand All @@ -157,6 +177,10 @@ func isProtocolAddr(a ma.Multiaddr, p int) bool {
return found
}

func isQUICAddr(a ma.Multiaddr) bool {
return isProtocolAddr(a, ma.P_QUIC) || isProtocolAddr(a, ma.P_QUIC_V1)
}

// filterAddrs filters an address slice in place
func filterAddrs(addrs []ma.Multiaddr, f func(a ma.Multiaddr) bool) (filtered, rest []ma.Multiaddr) {
j := 0
Expand Down
99 changes: 63 additions & 36 deletions p2p/net/swarm/dial_ranker_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ func sortAddrDelays(addrDelays []network.AddrDelay) {
})
}

func TestNoDelayRanker(t *testing.T) {
func TestNoDelayDialRanker(t *testing.T) {
q1 := ma.StringCast("/ip4/1.2.3.4/udp/1/quic")
q1v1 := ma.StringCast("/ip4/1.2.3.4/udp/1/quic-v1")
wt1 := ma.StringCast("/ip4/1.2.3.4/udp/1/quic-v1/webtransport/")
Expand All @@ -36,7 +36,7 @@ func TestNoDelayRanker(t *testing.T) {
output []network.AddrDelay
}{
{
name: "quic-ranking",
name: "quic+webtransport filtered when quicv1",
addrs: []ma.Multiaddr{q1, q2, q3, q4, q1v1, q2v1, q3v1, wt1, t1},
output: []network.AddrDelay{
{Addr: q1, Delay: 0},
Expand Down Expand Up @@ -89,7 +89,7 @@ func TestDelayRankerQUICDelay(t *testing.T) {
output []network.AddrDelay
}{
{
name: "single quic dialed first",
name: "quic-ipv4",
addrs: []ma.Multiaddr{q1, q2, q3, q4},
output: []network.AddrDelay{
{Addr: q1, Delay: 0},
Expand All @@ -99,37 +99,49 @@ func TestDelayRankerQUICDelay(t *testing.T) {
},
},
{
name: "quicv1 dialed before quic",
addrs: []ma.Multiaddr{q1, q2v1, q3, q4},
name: "quic-ipv6",
addrs: []ma.Multiaddr{q1v16, q2v16, q3v16},
output: []network.AddrDelay{
{Addr: q2v1, Delay: 0},
{Addr: q1, Delay: PublicQUICDelay},
{Addr: q3, Delay: PublicQUICDelay},
{Addr: q4, Delay: PublicQUICDelay},
{Addr: q1v16, Delay: 0},
{Addr: q2v16, Delay: PublicQUICDelay},
{Addr: q3v16, Delay: PublicQUICDelay},
},
},
{
name: "quic-quic-v1-webtransport",
addrs: []ma.Multiaddr{q1, q2, q3, q4, q1v1, q2v1, q3v1, wt1},
name: "quic-ip4-ip6",
addrs: []ma.Multiaddr{q1, q1v16, q2v1, q3, q4},
output: []network.AddrDelay{
{Addr: q1v1, Delay: 0},
{Addr: q1, Delay: PublicQUICDelay},
{Addr: q2, Delay: PublicQUICDelay},
{Addr: q3, Delay: PublicQUICDelay},
{Addr: q4, Delay: PublicQUICDelay},
{Addr: q1v16, Delay: 0},
{Addr: q2v1, Delay: PublicQUICDelay},
{Addr: q3v1, Delay: PublicQUICDelay},
{Addr: wt1, Delay: PublicQUICDelay},
{Addr: q1, Delay: 2 * PublicQUICDelay},
{Addr: q3, Delay: 2 * PublicQUICDelay},
{Addr: q4, Delay: 2 * PublicQUICDelay},
},
},
{
name: "ipv6",
addrs: []ma.Multiaddr{q1v16, q2v16, q3v16, q1},
name: "quic-quic-v1-webtransport",
addrs: []ma.Multiaddr{q1v16, q1, q2, q3, q4, q1v1, q2v1, q3v1, wt1},
output: []network.AddrDelay{
{Addr: q1, Delay: 0},
{Addr: q1v16, Delay: 0},
{Addr: q2v16, Delay: PublicQUICDelay},
{Addr: q3v16, Delay: PublicQUICDelay},
{Addr: q1v1, Delay: PublicQUICDelay},
{Addr: q2v1, Delay: 2 * PublicQUICDelay},
{Addr: q3v1, Delay: 2 * PublicQUICDelay},
{Addr: q1, Delay: 2 * PublicQUICDelay},
{Addr: q2, Delay: 2 * PublicQUICDelay},
{Addr: q3, Delay: 2 * PublicQUICDelay},
{Addr: q4, Delay: 2 * PublicQUICDelay},
{Addr: wt1, Delay: 2 * PublicQUICDelay},
},
},
{
name: "wt-ranking",
addrs: []ma.Multiaddr{q1v16, q2v16, q3v16, q2, wt1},
output: []network.AddrDelay{
{Addr: q1v16, Delay: 0},
{Addr: q2, Delay: PublicQUICDelay},
{Addr: wt1, Delay: 2 * PublicQUICDelay},
{Addr: q2v16, Delay: 2 * PublicQUICDelay},
{Addr: q3v16, Delay: 2 * PublicQUICDelay},
},
},
}
Expand All @@ -152,11 +164,18 @@ func TestDelayRankerQUICDelay(t *testing.T) {
}

func TestDelayRankerTCPDelay(t *testing.T) {

q1 := ma.StringCast("/ip4/1.2.3.4/udp/1/quic")
q1v1 := ma.StringCast("/ip4/1.2.3.4/udp/1/quic-v1")
q2 := ma.StringCast("/ip4/1.2.3.4/udp/2/quic")
q2v1 := ma.StringCast("/ip4/1.2.3.4/udp/2/quic-v1")
q3 := ma.StringCast("/ip4/1.2.3.4/udp/3/quic")

q1v16 := ma.StringCast("/ip6/1::2/udp/1/quic-v1")
q2v16 := ma.StringCast("/ip6/1::2/udp/2/quic-v1")
q3v16 := ma.StringCast("/ip6/1::2/udp/3/quic-v1")

t1 := ma.StringCast("/ip4/1.2.3.5/tcp/1/")
t1v6 := ma.StringCast("/ip6/1::2/tcp/1")
t2 := ma.StringCast("/ip4/1.2.3.4/tcp/2")

testCase := []struct {
Expand All @@ -165,28 +184,36 @@ func TestDelayRankerTCPDelay(t *testing.T) {
output []network.AddrDelay
}{
{
name: "2 quic with tcp",
addrs: []ma.Multiaddr{q1, q2v1, t1, t2},
name: "quic-with-tcp-ip6-ip4",
addrs: []ma.Multiaddr{q1, q1v1, q1v16, q2v16, q3v16, q2v1, t1, t2},
output: []network.AddrDelay{
{Addr: q2v1, Delay: 0},
{Addr: q1, Delay: PublicQUICDelay},
{Addr: t1, Delay: PublicQUICDelay + PublicTCPDelay},
{Addr: t2, Delay: PublicQUICDelay + PublicTCPDelay},
{Addr: q1v16, Delay: 0},
{Addr: q1v1, Delay: PublicQUICDelay},
{Addr: q1, Delay: 2 * PublicQUICDelay},
{Addr: q2v16, Delay: 2 * PublicQUICDelay},
{Addr: q3v16, Delay: 2 * PublicQUICDelay},
{Addr: q2v1, Delay: 2 * PublicQUICDelay},
{Addr: t1, Delay: 3 * PublicQUICDelay},
{Addr: t2, Delay: 3 * PublicQUICDelay},
},
},
{
name: "1 quic with tcp",
addrs: []ma.Multiaddr{q1, t1, t2},
name: "quic-ip4-with-tcp",
addrs: []ma.Multiaddr{q1, q2, q3, t1, t2, t1v6},
output: []network.AddrDelay{
{Addr: q1, Delay: 0},
{Addr: t1, Delay: PublicTCPDelay},
{Addr: t2, Delay: PublicTCPDelay},
{Addr: q2, Delay: PublicQUICDelay},
{Addr: q3, Delay: PublicQUICDelay},
{Addr: t1, Delay: PublicQUICDelay + PublicTCPDelay},
{Addr: t2, Delay: PublicQUICDelay + PublicTCPDelay},
{Addr: t1v6, Delay: PublicQUICDelay + PublicTCPDelay},
},
},
{
name: "no quic",
addrs: []ma.Multiaddr{t1, t2},
name: "tcp-ip4-ip6",
addrs: []ma.Multiaddr{t1, t2, t1v6},
output: []network.AddrDelay{
{Addr: t1v6, Delay: 0},
{Addr: t1, Delay: 0},
{Addr: t2, Delay: 0},
},
Expand Down

0 comments on commit 9bf1c22

Please sign in to comment.