Ensure remotedialer kubelet connections use kubelet bind address

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>

go.mod

@@ -18,6 +18,7 @@ replace (
 	github.com/opencontainers/selinux => github.com/opencontainers/selinux v1.11.0
 	github.com/prometheus/client_golang => github.com/prometheus/client_golang v1.18.0
 	github.com/prometheus/common => github.com/prometheus/common v0.45.0
+	github.com/rancher/remotedialer => github.com/brandond/remotedialer v0.0.0-20240702173222-06553032edda
 	github.com/spegel-org/spegel => github.com/k3s-io/spegel v0.0.23-0.20240516234953-f3d2c4072314
 	github.com/ugorji/go => github.com/ugorji/go v1.2.11
 	go.etcd.io/etcd/api/v3 => github.com/k3s-io/etcd/api/v3 v3.5.13-k3s1

go.sum

@@ -307,6 +307,8 @@ github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2y
 github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
 github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
 github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
+github.com/brandond/remotedialer v0.0.0-20240702173222-06553032edda h1:ReuZw9o6xuXSZKdGEzHo7AYuPMGA+ZR9d143EL2TlQU=
+github.com/brandond/remotedialer v0.0.0-20240702173222-06553032edda/go.mod h1:Ys004RpJuTLSm+k4aYUCoFiOOad37ubYev3TkOFg/5w=
 github.com/bronze1man/goStrongswanVici v0.0.0-20221114103242-3f6dc524986c h1:JY0NMX6F455gODxoTSesio3OmhM8HEniyFMWgpHvTY0=
 github.com/bronze1man/goStrongswanVici v0.0.0-20221114103242-3f6dc524986c/go.mod h1:fWUtBEPt2yjrr3WFhOqvajM8JSEU8bEeBcoeSCsKRpc=
 github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
@@ -1405,8 +1407,6 @@ github.com/rancher/lasso v0.0.0-20240430201833-6f3def65ffc5 h1:6K4RhfmCy7uxaw9Oz
 github.com/rancher/lasso v0.0.0-20240430201833-6f3def65ffc5/go.mod h1:7WkdfPEvWAdnHVioMUkhpZkshJzjDY62ocHVhcbw89M=
 github.com/rancher/permissions v0.0.0-20240523180510-4001d3d637f7 h1:0Kg2SGoMeU1ll4xPi4DE0+qNHLFO/U5MwtK0WrIdK+o=
 github.com/rancher/permissions v0.0.0-20240523180510-4001d3d637f7/go.mod h1:fsbs0YOsGn1ofPD5p+BuI4qDhbMbSJtTegKt6Ucna+c=
-github.com/rancher/remotedialer v0.3.0 h1:y1EO8JCsgZo0RcqTUp6U8FXcBAv27R+TLnWRcpvX1sM=
-github.com/rancher/remotedialer v0.3.0/go.mod h1:BwwztuvViX2JrLLUwDlsYt5DiyUwHLlzynRwkZLAY0Q=
 github.com/rancher/wharfie v0.6.4 h1:JwYB+q661n8ut/ysgsjKe0P0z6bHCCFoC+29995ME90=
 github.com/rancher/wharfie v0.6.4/go.mod h1:kWv97z0sMAbnVNT/oe+JFZJVKn4xkas7ZdFf6UifWis=
 github.com/rancher/wrangler/v3 v3.0.0-rc2 h1:XGSPPp6GXELqlLvwJp5MsdqyCPu6SCA4UKJ7rQJzE40=
@@ -1423,8 +1423,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
-github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
-github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
+github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
+github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
 github.com/rootless-containers/rootlesskit v1.0.1 h1:jepqW1txFSowKSMAEkVhWH3Oa1TCY9S400MVYe/6Iro=
 github.com/rootless-containers/rootlesskit v1.0.1/go.mod h1:t2UAiYagxrJ+wmpFAUIZPcqsm4k2B7ve6g7lILKbloc=
 github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=

pkg/agent/tunnel/tunnel.go

@@ -38,13 +38,15 @@ import (
 
 var (
 	endpointDebounceDelay = time.Second
+	defaultDialer         = net.Dialer{}
 )
 
 type agentTunnel struct {
 	client      kubernetes.Interface
 	cidrs       cidranger.Ranger
 	ports       map[string]bool
 	mode        string
+	kubeletAddr string
 	kubeletPort string
 	startTime   time.Time
 }
@@ -82,6 +84,7 @@ func Setup(ctx context.Context, config *daemonconfig.Node, proxy proxy.Proxy) er
 		cidrs:       cidranger.NewPCTrieRanger(),
 		ports:       map[string]bool{},
 		mode:        config.EgressSelectorMode,
+		kubeletAddr: config.AgentConfig.ListenAddress,
 		kubeletPort: fmt.Sprint(ports.KubeletPort),
 		startTime:   time.Now().Truncate(time.Second),
 	}
@@ -186,7 +189,7 @@ func (a *agentTunnel) setKubeletPort(ctx context.Context, apiServerReady <-chan
 			return false, nil
 		}
 		a.kubeletPort = kubeletPort
-		logrus.Infof("Tunnel authorizer set Kubelet Port %s", a.kubeletPort)
+		logrus.Infof("Tunnel authorizer set Kubelet Port %s", net.JoinHostPort(a.kubeletAddr, a.kubeletPort))
 		return true, nil
 	})
 }
@@ -390,7 +393,7 @@ func (a *agentTunnel) authorized(ctx context.Context, proto, address string) boo
 	logrus.Debugf("Tunnel authorizer checking dial request for %s", address)
 	host, port, err := net.SplitHostPort(address)
 	if err == nil {
-		if a.isKubeletPort(proto, host, port) {
+		if a.isKubeletOrStreamPort(proto, host, port) {
 			return true
 		}
 		if ip := net.ParseIP(host); ip != nil {
@@ -448,7 +451,7 @@ func (a *agentTunnel) connect(rootCtx context.Context, waitGroup *sync.WaitGroup
 	go func() {
 		for {
 			// ConnectToProxy blocks until error or context cancellation
-			err := remotedialer.ConnectToProxy(ctx, wsURL, nil, auth, ws, onConnect)
+			err := remotedialer.ConnectToProxyWithDialer(ctx, wsURL, nil, auth, ws, a.dialContext, onConnect)
 			connected = false
 			if err != nil && !errors.Is(err, context.Canceled) {
 				logrus.WithField("url", wsURL).WithError(err).Error("Remotedialer proxy error; reconnecting...")
@@ -471,7 +474,21 @@ func (a *agentTunnel) connect(rootCtx context.Context, waitGroup *sync.WaitGroup
 	}
 }
 
-// isKubeletPort returns true if the connection is to a reserved TCP port on a loopback address.
-func (a *agentTunnel) isKubeletPort(proto, host, port string) bool {
+// isKubeletOrStreamPort returns true if the connection is to a reserved TCP port on a loopback address.
+func (a *agentTunnel) isKubeletOrStreamPort(proto, host, port string) bool {
 	return proto == "tcp" && (host == "127.0.0.1" || host == "::1") && (port == a.kubeletPort || port == daemonconfig.StreamServerPort)
 }
+
+// dialContext dials a local connection on behalf of the remote server.  If the
+// connection is to the kubelet port on the loopback address, the kubelet is dialed
+// at its configured bind address.  Otherwise, the connection is dialed normally.
+func (a *agentTunnel) dialContext(ctx context.Context, network, address string) (net.Conn, error) {
+	host, port, err := net.SplitHostPort(address)
+	if err != nil {
+		return nil, err
+	}
+	if a.isKubeletOrStreamPort(network, host, port) && port == a.kubeletPort {
+		address = net.JoinHostPort(a.kubeletAddr, port)
+	}
+	return defaultDialer.DialContext(ctx, network, address)
+}

pkg/daemons/control/tunnel.go

@@ -3,7 +3,6 @@ package control
 import (
 	"bufio"
 	"context"
-	"fmt"
 	"io"
 	"net"
 	"net/http"
@@ -197,7 +196,6 @@ func (t *TunnelServer) dialBackend(ctx context.Context, addr string) (net.Conn,
 	if err != nil {
 		return nil, err
 	}
-	loopback := t.config.Loopback(true)
 
 	var nodeName string
 	var toKubelet, useTunnel bool
@@ -224,14 +222,17 @@ func (t *TunnelServer) dialBackend(ctx context.Context, addr string) (net.Conn,
 		useTunnel = true
 	}
 
-	// Always dial kubelet via the loopback address.
-	if toKubelet {
-		addr = fmt.Sprintf("%s:%s", loopback, port)
-	}
-
 	// If connecting to something hosted by the local node, don't tunnel
 	if nodeName == t.config.ServerNodeName {
 		useTunnel = false
+		if toKubelet {
+			// Dial local kubelet at the configured bind address
+			addr = net.JoinHostPort(t.config.BindAddress, port)
+		}
+	} else if toKubelet {
+		// Dial remote kubelet via the loopback address, the remotedialer client
+		// will ensure that it hits the right local address.
+		addr = net.JoinHostPort(t.config.Loopback(false), port)
 	}
 
 	if useTunnel {