From 526cb89d060e78de2b3d22af21fbb67c1635bc54 Mon Sep 17 00:00:00 2001
From: cce <51567+cce@users.noreply.github.com>
Date: Thu, 17 Nov 2022 11:45:28 -0500
Subject: [PATCH] telemetry: add TCP RTT info collection (#4745)
---
logging/telemetryspec/event.go | 4 +
.../limitlistener/rejectingLimitListener.go | 4 +
network/requestTracker.go | 4 +
network/wsNetwork.go | 24 +-
network/wsNetwork_test.go | 218 +++++++++---------
network/wsPeer.go | 5 +
util/tcpinfo.go | 72 ++++++
util/tcpinfo_darwin.go | 49 ++++
util/tcpinfo_linux.go | 129 +++++++++++
util/tcpinfo_noop.go | 26 +++
10 files changed, 423 insertions(+), 112 deletions(-)
create mode 100644 util/tcpinfo.go
create mode 100644 util/tcpinfo_darwin.go
create mode 100644 util/tcpinfo_linux.go
create mode 100644 util/tcpinfo_noop.go
diff --git a/logging/telemetryspec/event.go b/logging/telemetryspec/event.go
index 91c7bddf90..b98830bc0e 100644
--- a/logging/telemetryspec/event.go
+++ b/logging/telemetryspec/event.go
@@ -18,6 +18,8 @@ package telemetryspec
import (
"time"
+
+ "github.com/algorand/go-algorand/util"
)
// Telemetry Events
@@ -302,6 +304,8 @@ type PeerConnectionDetails struct {
MessageDelay int64 `json:",omitempty"`
// DuplicateFilterCount is the number of times this peer has sent us a message hash to filter that it had already sent before.
DuplicateFilterCount uint64
+ // TCPInfo provides connection measurements from TCP.
+ TCP util.TCPInfo `json:",omitempty"`
}
// CatchpointGenerationEvent event
diff --git a/network/limitlistener/rejectingLimitListener.go b/network/limitlistener/rejectingLimitListener.go
index 60d89199c8..9d6a5914bb 100644
--- a/network/limitlistener/rejectingLimitListener.go
+++ b/network/limitlistener/rejectingLimitListener.go
@@ -83,3 +83,7 @@ func (l *rejectingLimitListenerConn) Close() error {
l.releaseOnce.Do(l.release)
return err
}
+
+func (l *rejectingLimitListenerConn) UnderlyingConn() net.Conn {
+ return l.Conn
+}
diff --git a/network/requestTracker.go b/network/requestTracker.go
index fd78dadca1..2ae34c81f1 100644
--- a/network/requestTracker.go
+++ b/network/requestTracker.go
@@ -254,6 +254,10 @@ type requestTrackedConnection struct {
tracker *RequestTracker
}
+func (c *requestTrackedConnection) UnderlyingConn() net.Conn {
+ return c.Conn
+}
+
// Close removes the connection from the tracker's connections map and call the underlaying Close function.
func (c *requestTrackedConnection) Close() error {
c.tracker.hostRequestsMu.Lock()
diff --git a/network/wsNetwork.go b/network/wsNetwork.go
index d5f8b111e8..2dbe9b5908 100644
--- a/network/wsNetwork.go
+++ b/network/wsNetwork.go
@@ -1805,14 +1805,23 @@ func (wn *WebsocketNetwork) OnNetworkAdvance() {
// to the telemetry server. Internally, it's using a timer to ensure that it would only
// send the information once every hour ( configurable via PeerConnectionsUpdateInterval )
func (wn *WebsocketNetwork) sendPeerConnectionsTelemetryStatus() {
+ if !wn.log.GetTelemetryEnabled() {
+ return
+ }
now := time.Now()
if wn.lastPeerConnectionsSent.Add(time.Duration(wn.config.PeerConnectionsUpdateInterval)*time.Second).After(now) || wn.config.PeerConnectionsUpdateInterval <= 0 {
// it's not yet time to send the update.
return
}
wn.lastPeerConnectionsSent = now
+
var peers []*wsPeer
peers, _ = wn.peerSnapshot(peers)
+ connectionDetails := wn.getPeerConnectionTelemetryDetails(now, peers)
+ wn.log.EventWithDetails(telemetryspec.Network, telemetryspec.PeerConnectionsEvent, connectionDetails)
+}
+
+func (wn *WebsocketNetwork) getPeerConnectionTelemetryDetails(now time.Time, peers []*wsPeer) telemetryspec.PeersConnectionDetails {
var connectionDetails telemetryspec.PeersConnectionDetails
for _, peer := range peers {
connDetail := telemetryspec.PeerConnectionDetails{
@@ -1821,6 +1830,18 @@ func (wn *WebsocketNetwork) sendPeerConnectionsTelemetryStatus() {
InstanceName: peer.InstanceName,
DuplicateFilterCount: peer.duplicateFilterCount,
}
+ // unwrap websocket.Conn, requestTrackedConnection, rejectingLimitListenerConn
+ var uconn net.Conn = peer.conn.UnderlyingConn()
+ for i := 0; i < 10; i++ {
+ wconn, ok := uconn.(wrappedConn)
+ if !ok {
+ break
+ }
+ uconn = wconn.UnderlyingConn()
+ }
+ if tcpInfo, err := util.GetConnTCPInfo(uconn); err == nil && tcpInfo != nil {
+ connDetail.TCP = *tcpInfo
+ }
if peer.outgoing {
connDetail.Address = justHost(peer.conn.RemoteAddr().String())
connDetail.Endpoint = peer.GetAddress()
@@ -1831,8 +1852,7 @@ func (wn *WebsocketNetwork) sendPeerConnectionsTelemetryStatus() {
connectionDetails.IncomingPeers = append(connectionDetails.IncomingPeers, connDetail)
}
}
-
- wn.log.EventWithDetails(telemetryspec.Network, telemetryspec.PeerConnectionsEvent, connectionDetails)
+ return connectionDetails
}
// prioWeightRefreshTime controls how often we refresh the weights
diff --git a/network/wsNetwork_test.go b/network/wsNetwork_test.go
index 37dd646aa3..c2e4678595 100644
--- a/network/wsNetwork_test.go
+++ b/network/wsNetwork_test.go
@@ -20,6 +20,7 @@ import (
"bytes"
"context"
"encoding/binary"
+ "encoding/json"
"fmt"
"io"
"math/rand"
@@ -268,14 +269,17 @@ func netStop(t testing.TB, wn *WebsocketNetwork, name string) {
t.Logf("%s done", name)
}
-// Set up two nodes, test that a.Broadcast is received by B
-func TestWebsocketNetworkBasic(t *testing.T) {
- partitiontest.PartitionTest(t)
+func setupWebsocketNetworkAB(t *testing.T, countTarget int) (*WebsocketNetwork, *WebsocketNetwork, *messageCounterHandler, func()) {
+ success := false
netA := makeTestWebsocketNode(t)
netA.config.GossipFanout = 1
netA.Start()
- defer netStop(t, netA, "A")
+ defer func() {
+ if !success {
+ netStop(t, netA, "A")
+ }
+ }()
netB := makeTestWebsocketNode(t)
netB.config.GossipFanout = 1
addrA, postListen := netA.Address()
@@ -283,9 +287,12 @@ func TestWebsocketNetworkBasic(t *testing.T) {
t.Log(addrA)
netB.phonebook.ReplacePeerList([]string{addrA}, "default", PhoneBookEntryRelayRole)
netB.Start()
- defer netStop(t, netB, "B")
- counter := newMessageCounter(t, 2)
- counterDone := counter.done
+ defer func() {
+ if !success {
+ netStop(t, netB, "B")
+ }
+ }()
+ counter := newMessageCounter(t, countTarget)
netB.RegisterHandlers([]TaggedMessageHandler{{Tag: protocol.TxnTag, MessageHandler: counter}})
readyTimeout := time.NewTimer(2 * time.Second)
@@ -294,6 +301,21 @@ func TestWebsocketNetworkBasic(t *testing.T) {
waitReady(t, netB, readyTimeout.C)
t.Log("b ready")
+ success = true
+ closeFunc := func() {
+ netStop(t, netB, "B")
+ netStop(t, netB, "A")
+ }
+ return netA, netB, counter, closeFunc
+}
+
+// Set up two nodes, test that a.Broadcast is received by B
+func TestWebsocketNetworkBasic(t *testing.T) {
+ partitiontest.PartitionTest(t)
+
+ netA, _, counter, closeFunc := setupWebsocketNetworkAB(t, 2)
+ defer closeFunc()
+ counterDone := counter.done
netA.Broadcast(context.Background(), protocol.TxnTag, []byte("foo"), false, nil)
netA.Broadcast(context.Background(), protocol.TxnTag, []byte("bar"), false, nil)
@@ -384,27 +406,9 @@ func TestWebsocketProposalPayloadCompression(t *testing.T) {
func TestWebsocketNetworkUnicast(t *testing.T) {
partitiontest.PartitionTest(t)
- netA := makeTestWebsocketNode(t)
- netA.config.GossipFanout = 1
- netA.Start()
- defer netStop(t, netA, "A")
- netB := makeTestWebsocketNode(t)
- netB.config.GossipFanout = 1
- addrA, postListen := netA.Address()
- require.True(t, postListen)
- t.Log(addrA)
- netB.phonebook.ReplacePeerList([]string{addrA}, "default", PhoneBookEntryRelayRole)
- netB.Start()
- defer netStop(t, netB, "B")
- counter := newMessageCounter(t, 2)
+ netA, _, counter, closeFunc := setupWebsocketNetworkAB(t, 2)
+ defer closeFunc()
counterDone := counter.done
- netB.RegisterHandlers([]TaggedMessageHandler{{Tag: protocol.TxnTag, MessageHandler: counter}})
-
- readyTimeout := time.NewTimer(2 * time.Second)
- waitReady(t, netA, readyTimeout.C)
- t.Log("a ready")
- waitReady(t, netB, readyTimeout.C)
- t.Log("b ready")
require.Equal(t, 1, len(netA.peers))
require.Equal(t, 1, len(netA.GetPeers(PeersConnectedIn)))
@@ -425,26 +429,8 @@ func TestWebsocketNetworkUnicast(t *testing.T) {
func TestWebsocketPeerData(t *testing.T) {
partitiontest.PartitionTest(t)
- netA := makeTestWebsocketNode(t)
- netA.config.GossipFanout = 1
- netA.Start()
- defer netStop(t, netA, "A")
- netB := makeTestWebsocketNode(t)
- netB.config.GossipFanout = 1
- addrA, postListen := netA.Address()
- require.True(t, postListen)
- t.Log(addrA)
- netB.phonebook.ReplacePeerList([]string{addrA}, "default", PhoneBookEntryRelayRole)
- netB.Start()
- defer netStop(t, netB, "B")
- counter := newMessageCounter(t, 2)
- netB.RegisterHandlers([]TaggedMessageHandler{{Tag: protocol.TxnTag, MessageHandler: counter}})
-
- readyTimeout := time.NewTimer(2 * time.Second)
- waitReady(t, netA, readyTimeout.C)
- t.Log("a ready")
- waitReady(t, netB, readyTimeout.C)
- t.Log("b ready")
+ netA, _, _, closeFunc := setupWebsocketNetworkAB(t, 2)
+ defer closeFunc()
require.Equal(t, 1, len(netA.peers))
require.Equal(t, 1, len(netA.GetPeers(PeersConnectedIn)))
@@ -463,27 +449,9 @@ func TestWebsocketPeerData(t *testing.T) {
func TestWebsocketNetworkArray(t *testing.T) {
partitiontest.PartitionTest(t)
- netA := makeTestWebsocketNode(t)
- netA.config.GossipFanout = 1
- netA.Start()
- defer netStop(t, netA, "A")
- netB := makeTestWebsocketNode(t)
- netB.config.GossipFanout = 1
- addrA, postListen := netA.Address()
- require.True(t, postListen)
- t.Log(addrA)
- netB.phonebook.ReplacePeerList([]string{addrA}, "default", PhoneBookEntryRelayRole)
- netB.Start()
- defer netStop(t, netB, "B")
- counter := newMessageCounter(t, 3)
+ netA, _, counter, closeFunc := setupWebsocketNetworkAB(t, 3)
+ defer closeFunc()
counterDone := counter.done
- netB.RegisterHandlers([]TaggedMessageHandler{{Tag: protocol.TxnTag, MessageHandler: counter}})
-
- readyTimeout := time.NewTimer(2 * time.Second)
- waitReady(t, netA, readyTimeout.C)
- t.Log("a ready")
- waitReady(t, netB, readyTimeout.C)
- t.Log("b ready")
tags := []protocol.Tag{protocol.TxnTag, protocol.TxnTag, protocol.TxnTag}
data := [][]byte{[]byte("foo"), []byte("bar"), []byte("algo")}
@@ -500,27 +468,9 @@ func TestWebsocketNetworkArray(t *testing.T) {
func TestWebsocketNetworkCancel(t *testing.T) {
partitiontest.PartitionTest(t)
- netA := makeTestWebsocketNode(t)
- netA.config.GossipFanout = 1
- netA.Start()
- defer netStop(t, netA, "A")
- netB := makeTestWebsocketNode(t)
- netB.config.GossipFanout = 1
- addrA, postListen := netA.Address()
- require.True(t, postListen)
- t.Log(addrA)
- netB.phonebook.ReplacePeerList([]string{addrA}, "default", PhoneBookEntryRelayRole)
- netB.Start()
- defer netStop(t, netB, "B")
- counter := newMessageCounter(t, 100)
+ netA, _, counter, closeFunc := setupWebsocketNetworkAB(t, 100)
+ defer closeFunc()
counterDone := counter.done
- netB.RegisterHandlers([]TaggedMessageHandler{{Tag: protocol.TxnTag, MessageHandler: counter}})
-
- readyTimeout := time.NewTimer(2 * time.Second)
- waitReady(t, netA, readyTimeout.C)
- t.Log("a ready")
- waitReady(t, netB, readyTimeout.C)
- t.Log("b ready")
tags := make([]protocol.Tag, 100)
data := make([][]byte, 100)
@@ -721,29 +671,15 @@ func TestAddrToGossipAddr(t *testing.T) {
type nopConn struct{}
-func (nc *nopConn) RemoteAddr() net.Addr {
- return nil
-}
-func (nc *nopConn) NextReader() (int, io.Reader, error) {
- return 0, nil, nil
-}
-func (nc *nopConn) WriteMessage(int, []byte) error {
- return nil
-}
-func (nc *nopConn) WriteControl(int, []byte, time.Time) error {
- return nil
-}
-func (nc *nopConn) SetReadLimit(limit int64) {
-}
-func (nc *nopConn) CloseWithoutFlush() error {
- return nil
-}
-func (nc *nopConn) SetPingHandler(h func(appData string) error) {
-
-}
-func (nc *nopConn) SetPongHandler(h func(appData string) error) {
-
-}
+func (nc *nopConn) RemoteAddr() net.Addr { return nil }
+func (nc *nopConn) NextReader() (int, io.Reader, error) { return 0, nil, nil }
+func (nc *nopConn) WriteMessage(int, []byte) error { return nil }
+func (nc *nopConn) WriteControl(int, []byte, time.Time) error { return nil }
+func (nc *nopConn) SetReadLimit(limit int64) {}
+func (nc *nopConn) CloseWithoutFlush() error { return nil }
+func (nc *nopConn) SetPingHandler(h func(appData string) error) {}
+func (nc *nopConn) SetPongHandler(h func(appData string) error) {}
+func (nc *nopConn) UnderlyingConn() net.Conn { return nil }
var nopConnSingleton = nopConn{}
@@ -2739,3 +2675,65 @@ func TestPreparePeerData(t *testing.T) {
}
}
}
+
+func TestWebsocketNetworkTelemetryTCP(t *testing.T) {
+ partitiontest.PartitionTest(t)
+
+ // start two networks and send 2 messages from A to B
+ closed := false
+ netA, netB, counter, closeFunc := setupWebsocketNetworkAB(t, 2)
+ defer func() {
+ if !closed {
+ closeFunc()
+ }
+ }()
+ counterDone := counter.done
+ netA.Broadcast(context.Background(), protocol.TxnTag, []byte("foo"), false, nil)
+ netA.Broadcast(context.Background(), protocol.TxnTag, []byte("bar"), false, nil)
+
+ select {
+ case <-counterDone:
+ case <-time.After(2 * time.Second):
+ t.Errorf("timeout, count=%d, wanted 2", counter.count)
+ }
+
+ // get RTT from both ends and assert nonzero
+ var peersA, peersB []*wsPeer
+ peersA, _ = netA.peerSnapshot(peersA)
+ detailsA := netA.getPeerConnectionTelemetryDetails(time.Now(), peersA)
+ peersB, _ = netB.peerSnapshot(peersB)
+ detailsB := netB.getPeerConnectionTelemetryDetails(time.Now(), peersB)
+ require.Len(t, detailsA.IncomingPeers, 1)
+ assert.NotZero(t, detailsA.IncomingPeers[0].TCP.RTT)
+ require.Len(t, detailsB.OutgoingPeers, 1)
+ assert.NotZero(t, detailsB.OutgoingPeers[0].TCP.RTT)
+
+ pcdA, err := json.Marshal(detailsA)
+ assert.NoError(t, err)
+ pcdB, err := json.Marshal(detailsB)
+ assert.NoError(t, err)
+ t.Log("detailsA", string(pcdA))
+ t.Log("detailsB", string(pcdB))
+
+ // close connections
+ closeFunc()
+ closed = true
+ // open more FDs by starting 2 more networks
+ _, _, _, closeFunc2 := setupWebsocketNetworkAB(t, 2)
+ defer closeFunc2()
+ // use stale peers snapshot from closed networks to get telemetry
+ // *net.OpError "use of closed network connection" err results in 0 rtt values
+ detailsA = netA.getPeerConnectionTelemetryDetails(time.Now(), peersA)
+ detailsB = netB.getPeerConnectionTelemetryDetails(time.Now(), peersB)
+ require.Len(t, detailsA.IncomingPeers, 1)
+ assert.Zero(t, detailsA.IncomingPeers[0].TCP.RTT)
+ require.Len(t, detailsB.OutgoingPeers, 1)
+ assert.Zero(t, detailsB.OutgoingPeers[0].TCP.RTT)
+
+ pcdA, err = json.Marshal(detailsA)
+ assert.NoError(t, err)
+ pcdB, err = json.Marshal(detailsB)
+ assert.NoError(t, err)
+ t.Log("closed detailsA", string(pcdA))
+ t.Log("closed detailsB", string(pcdB))
+}
diff --git a/network/wsPeer.go b/network/wsPeer.go
index e1ac6ffb31..d551cf1a66 100644
--- a/network/wsPeer.go
+++ b/network/wsPeer.go
@@ -124,6 +124,11 @@ type wsPeerWebsocketConn interface {
CloseWithoutFlush() error
SetPingHandler(h func(appData string) error)
SetPongHandler(h func(appData string) error)
+ wrappedConn
+}
+
+type wrappedConn interface {
+ UnderlyingConn() net.Conn
}
type sendMessage struct {
diff --git a/util/tcpinfo.go b/util/tcpinfo.go
new file mode 100644
index 0000000000..2b4c69d294
--- /dev/null
+++ b/util/tcpinfo.go
@@ -0,0 +1,72 @@
+// Copyright (C) 2019-2022 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package util
+
+import (
+ "errors"
+ "net"
+ "syscall"
+)
+
+// TCPInfo provides socket-level TCP information.
+type TCPInfo struct {
+ RTT uint32 `json:",omitempty"` // smoothed RTT
+ RTTVar uint32 `json:",omitempty"` // RTT variance
+ RTTMin uint32 `json:",omitempty"` // smallest observed RTT on the connection
+ SndMSS, RcvMSS uint32 `json:",omitempty"` // send and receive maximum segment size
+ SndCwnd uint32 `json:",omitempty"` // sender congestion window
+ SndWnd uint32 `json:",omitempty"` // send window advertised to receiver
+ RcvWnd uint32 `json:",omitempty"` // receive window advertised to sender
+ // tcpi_delivery_rate: The most recent goodput, as measured by
+ // tcp_rate_gen(). If the socket is limited by the sending
+ // application (e.g., no data to send), it reports the highest
+ // measurement instead of the most recent. The unit is bytes per
+ // second (like other rate fields in tcp_info).
+ Rate uint64 `json:",omitempty"`
+ // tcpi_delivery_rate_app_limited: A boolean indicating if the goodput
+ // was measured when the socket's throughput was limited by the
+ // sending application.
+ AppLimited bool `json:",omitempty"`
+}
+
+var (
+ // ErrNotSyscallConn is reported when GetConnTCPInfo is passed a connection that doesn't satisfy the syscall.Conn interface.
+ ErrNotSyscallConn = errors.New("conn doesn't satisfy syscall.Conn")
+ // ErrTCPInfoUnsupported is reported if TCP information is not available for this platform.
+ ErrTCPInfoUnsupported = errors.New("GetConnRTT not supported on this platform")
+ // ErrNoTCPInfo is reported if getsockopt returned no TCP info for some reason.
+ ErrNoTCPInfo = errors.New("getsockopt returned no TCP info")
+)
+
+// GetConnTCPInfo returns statistics for a TCP connection collected by the
+// underlying network implementation, using a system call on Linux and Mac
+// and returning an error for unsupported platforms.
+func GetConnTCPInfo(conn net.Conn) (*TCPInfo, error) {
+ if conn == nil {
+ return nil, ErrNotSyscallConn
+ }
+ sysconn, ok := conn.(syscall.Conn)
+ if sysconn == nil || !ok {
+ return nil, ErrNotSyscallConn
+ }
+ raw, err := sysconn.SyscallConn()
+ if err != nil {
+ return nil, err
+ }
+
+ return getConnTCPInfo(raw)
+}
diff --git a/util/tcpinfo_darwin.go b/util/tcpinfo_darwin.go
new file mode 100644
index 0000000000..cae19d06d4
--- /dev/null
+++ b/util/tcpinfo_darwin.go
@@ -0,0 +1,49 @@
+// Copyright (C) 2019-2022 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package util
+
+import (
+ "syscall"
+
+ "golang.org/x/sys/unix"
+)
+
+func getConnTCPInfo(raw syscall.RawConn) (*TCPInfo, error) {
+ var info *unix.TCPConnectionInfo
+ var getSockoptErr error
+ err := raw.Control(func(fd uintptr) {
+ info, getSockoptErr = unix.GetsockoptTCPConnectionInfo(int(fd), unix.IPPROTO_TCP, unix.TCP_CONNECTION_INFO)
+ })
+ if err != nil {
+ return nil, err
+ }
+ if getSockoptErr != nil {
+ return nil, getSockoptErr
+ }
+ if info == nil {
+ return nil, ErrNoTCPInfo
+ }
+ return &TCPInfo{
+ RTT: info.Srtt,
+ RTTVar: info.Rttvar,
+ SndMSS: info.Maxseg, // MSS is the same for snd/rcv according bsd/netinet/tcp_usrreq.c
+ RcvMSS: info.Maxseg,
+ SndCwnd: info.Snd_cwnd, // Send congestion window
+ SndWnd: info.Snd_wnd, // Advertised send window
+ RcvWnd: info.Rcv_wnd, // Advertised recv window
+ }, nil
+}
diff --git a/util/tcpinfo_linux.go b/util/tcpinfo_linux.go
new file mode 100644
index 0000000000..3da707e1cd
--- /dev/null
+++ b/util/tcpinfo_linux.go
@@ -0,0 +1,129 @@
+// Copyright (C) 2019-2022 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package util
+
+import (
+ "syscall"
+ "unsafe"
+
+ "golang.org/x/sys/unix"
+)
+
+func getConnTCPInfo(raw syscall.RawConn) (*TCPInfo, error) {
+ var info linuxTCPInfo
+ size := unsafe.Sizeof(info)
+
+ var errno syscall.Errno
+ err := raw.Control(func(fd uintptr) {
+ _, _, errno = unix.Syscall6(unix.SYS_GETSOCKOPT, fd, unix.IPPROTO_TCP, unix.TCP_INFO,
+ uintptr(unsafe.Pointer(&info)), uintptr(unsafe.Pointer(&size)), 0)
+ })
+ if err != nil {
+ return nil, err
+ }
+ if errno != 0 {
+ return nil, errno
+ }
+ if info == (linuxTCPInfo{}) {
+ return nil, ErrNoTCPInfo
+ }
+ return &TCPInfo{
+ RTT: info.rtt,
+ RTTVar: info.rttvar,
+ RTTMin: info.min_rtt,
+ SndMSS: info.snd_mss,
+ RcvMSS: info.rcv_mss,
+ SndCwnd: info.snd_cwnd, // Send congestion window
+ RcvWnd: info.snd_wnd, // "tp->snd_wnd, the receive window that the receiver has advertised to the sender."
+ Rate: info.delivery_rate,
+ AppLimited: bool((info.app_limited >> 7) != 0), // get first bit
+ }, nil
+}
+
+// linuxTCPInfo is based on linux include/uapi/linux/tcp.h struct tcp_info
+//revive:disable:var-naming
+type linuxTCPInfo struct {
+ state uint8
+ ca_state uint8
+ retransmits uint8
+ probes uint8
+ backoff uint8
+ options uint8
+ wscale uint8 // __u8 tcpi_snd_wscale : 4, tcpi_rcv_wscale : 4;
+ app_limited uint8 // __u8 tcpi_delivery_rate_app_limited:1, tcpi_fastopen_client_fail:2;
+
+ rto uint32
+ ato uint32
+ snd_mss uint32
+ rcv_mss uint32
+
+ unacked uint32
+ sacked uint32
+ lost uint32
+ retrans uint32
+ fackets uint32
+
+ last_data_sent uint32
+ last_ack_sent uint32
+ last_data_recv uint32
+ last_ack_recv uint32
+
+ pmtu uint32
+ rcv_ssthresh uint32
+ rtt uint32
+ rttvar uint32
+ snd_ssthresh uint32
+ snd_cwnd uint32
+ advmss uint32
+ reordering uint32
+
+ rcv_rtt uint32
+ rcv_space uint32
+
+ total_retrans uint32
+
+ // extended info beyond what's in syscall.TCPInfo
+ pacing_rate uint64
+ max_pacing_rate uint64
+ byte_acked uint64
+ bytes_received uint64
+ segs_out uint32
+ segs_in uint32
+
+ notsent_bytes uint32
+ min_rtt uint32
+ data_segs_in uint32
+ data_segs_out uint32
+
+ delivery_rate uint64
+
+ busy_time uint64
+ rwnd_limited uint64
+ sndbuf_limited uint64
+
+ delivered uint32
+ delivered_ce uint32
+
+ bytes_sent uint64
+ bytes_retrans uint64
+ dsack_dups uint32
+ reord_seen uint32
+
+ rcv_ooopack uint32
+
+ snd_wnd uint32
+}
diff --git a/util/tcpinfo_noop.go b/util/tcpinfo_noop.go
new file mode 100644
index 0000000000..a155ed9298
--- /dev/null
+++ b/util/tcpinfo_noop.go
@@ -0,0 +1,26 @@
+// Copyright (C) 2019-2022 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+//go:build !linux && !darwin
+// +build !linux,!darwin
+
+package util
+
+import "syscall"
+
+func getConnTCPInfo(conn syscall.RawConn) (*TCPInfo, error) {
+ return nil, ErrTCPInfoUnsupported
+}