From 5acd78738501462d3e5705d62557f9f07357b100 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Tue, 9 Jul 2024 23:36:29 +0000 Subject: [PATCH] Ensure remotedialer kubelet connections use kubelet bind address Signed-off-by: Brad Davidson --- go.mod | 1 + go.sum | 8 ++++---- pkg/agent/tunnel/tunnel.go | 27 ++++++++++++++++++++++----- pkg/daemons/control/tunnel.go | 15 ++++++++------- 4 files changed, 35 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index 4b82b0c8b202..44d8118257a3 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ replace ( github.com/opencontainers/selinux => github.com/opencontainers/selinux v1.11.0 github.com/prometheus/client_golang => github.com/prometheus/client_golang v1.18.0 github.com/prometheus/common => github.com/prometheus/common v0.45.0 + github.com/rancher/remotedialer => github.com/brandond/remotedialer v0.0.0-20240702173222-06553032edda github.com/spegel-org/spegel => github.com/k3s-io/spegel v0.0.23-0.20240516234953-f3d2c4072314 github.com/ugorji/go => github.com/ugorji/go v1.2.11 go.etcd.io/etcd/api/v3 => github.com/k3s-io/etcd/api/v3 v3.5.13-k3s1 diff --git a/go.sum b/go.sum index 73c579956300..f2fa814f9891 100644 --- a/go.sum +++ b/go.sum @@ -307,6 +307,8 @@ github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2y github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= +github.com/brandond/remotedialer v0.0.0-20240702173222-06553032edda h1:ReuZw9o6xuXSZKdGEzHo7AYuPMGA+ZR9d143EL2TlQU= +github.com/brandond/remotedialer v0.0.0-20240702173222-06553032edda/go.mod h1:Ys004RpJuTLSm+k4aYUCoFiOOad37ubYev3TkOFg/5w= github.com/bronze1man/goStrongswanVici v0.0.0-20221114103242-3f6dc524986c h1:JY0NMX6F455gODxoTSesio3OmhM8HEniyFMWgpHvTY0= github.com/bronze1man/goStrongswanVici v0.0.0-20221114103242-3f6dc524986c/go.mod h1:fWUtBEPt2yjrr3WFhOqvajM8JSEU8bEeBcoeSCsKRpc= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= @@ -1405,8 +1407,6 @@ github.com/rancher/lasso v0.0.0-20240430201833-6f3def65ffc5 h1:6K4RhfmCy7uxaw9Oz github.com/rancher/lasso v0.0.0-20240430201833-6f3def65ffc5/go.mod h1:7WkdfPEvWAdnHVioMUkhpZkshJzjDY62ocHVhcbw89M= github.com/rancher/permissions v0.0.0-20240523180510-4001d3d637f7 h1:0Kg2SGoMeU1ll4xPi4DE0+qNHLFO/U5MwtK0WrIdK+o= github.com/rancher/permissions v0.0.0-20240523180510-4001d3d637f7/go.mod h1:fsbs0YOsGn1ofPD5p+BuI4qDhbMbSJtTegKt6Ucna+c= -github.com/rancher/remotedialer v0.3.0 h1:y1EO8JCsgZo0RcqTUp6U8FXcBAv27R+TLnWRcpvX1sM= -github.com/rancher/remotedialer v0.3.0/go.mod h1:BwwztuvViX2JrLLUwDlsYt5DiyUwHLlzynRwkZLAY0Q= github.com/rancher/wharfie v0.6.4 h1:JwYB+q661n8ut/ysgsjKe0P0z6bHCCFoC+29995ME90= github.com/rancher/wharfie v0.6.4/go.mod h1:kWv97z0sMAbnVNT/oe+JFZJVKn4xkas7ZdFf6UifWis= github.com/rancher/wrangler/v3 v3.0.0-rc2 h1:XGSPPp6GXELqlLvwJp5MsdqyCPu6SCA4UKJ7rQJzE40= @@ -1423,8 +1423,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rootless-containers/rootlesskit v1.0.1 h1:jepqW1txFSowKSMAEkVhWH3Oa1TCY9S400MVYe/6Iro= github.com/rootless-containers/rootlesskit v1.0.1/go.mod h1:t2UAiYagxrJ+wmpFAUIZPcqsm4k2B7ve6g7lILKbloc= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= diff --git a/pkg/agent/tunnel/tunnel.go b/pkg/agent/tunnel/tunnel.go index 23c6dac404b8..79122c6b1f16 100644 --- a/pkg/agent/tunnel/tunnel.go +++ b/pkg/agent/tunnel/tunnel.go @@ -38,6 +38,7 @@ import ( var ( endpointDebounceDelay = time.Second + defaultDialer = net.Dialer{} ) type agentTunnel struct { @@ -45,6 +46,7 @@ type agentTunnel struct { cidrs cidranger.Ranger ports map[string]bool mode string + kubeletAddr string kubeletPort string startTime time.Time } @@ -82,6 +84,7 @@ func Setup(ctx context.Context, config *daemonconfig.Node, proxy proxy.Proxy) er cidrs: cidranger.NewPCTrieRanger(), ports: map[string]bool{}, mode: config.EgressSelectorMode, + kubeletAddr: config.AgentConfig.ListenAddress, kubeletPort: fmt.Sprint(ports.KubeletPort), startTime: time.Now().Truncate(time.Second), } @@ -186,7 +189,7 @@ func (a *agentTunnel) setKubeletPort(ctx context.Context, apiServerReady <-chan return false, nil } a.kubeletPort = kubeletPort - logrus.Infof("Tunnel authorizer set Kubelet Port %s", a.kubeletPort) + logrus.Infof("Tunnel authorizer set Kubelet Port %s", net.JoinHostPort(a.kubeletAddr, a.kubeletPort)) return true, nil }) } @@ -390,7 +393,7 @@ func (a *agentTunnel) authorized(ctx context.Context, proto, address string) boo logrus.Debugf("Tunnel authorizer checking dial request for %s", address) host, port, err := net.SplitHostPort(address) if err == nil { - if a.isKubeletPort(proto, host, port) { + if a.isKubeletOrStreamPort(proto, host, port) { return true } if ip := net.ParseIP(host); ip != nil { @@ -448,7 +451,7 @@ func (a *agentTunnel) connect(rootCtx context.Context, waitGroup *sync.WaitGroup go func() { for { // ConnectToProxy blocks until error or context cancellation - err := remotedialer.ConnectToProxy(ctx, wsURL, nil, auth, ws, onConnect) + err := remotedialer.ConnectToProxyWithDialer(ctx, wsURL, nil, auth, ws, a.dialContext, onConnect) connected = false if err != nil && !errors.Is(err, context.Canceled) { logrus.WithField("url", wsURL).WithError(err).Error("Remotedialer proxy error; reconnecting...") @@ -471,7 +474,21 @@ func (a *agentTunnel) connect(rootCtx context.Context, waitGroup *sync.WaitGroup } } -// isKubeletPort returns true if the connection is to a reserved TCP port on a loopback address. -func (a *agentTunnel) isKubeletPort(proto, host, port string) bool { +// isKubeletOrStreamPort returns true if the connection is to a reserved TCP port on a loopback address. +func (a *agentTunnel) isKubeletOrStreamPort(proto, host, port string) bool { return proto == "tcp" && (host == "127.0.0.1" || host == "::1") && (port == a.kubeletPort || port == daemonconfig.StreamServerPort) } + +// dialContext dials a local connection on behalf of the remote server. If the +// connection is to the kubelet port on the loopback address, the kubelet is dialed +// at its configured bind address. Otherwise, the connection is dialed normally. +func (a *agentTunnel) dialContext(ctx context.Context, network, address string) (net.Conn, error) { + host, port, err := net.SplitHostPort(address) + if err != nil { + return nil, err + } + if a.isKubeletOrStreamPort(network, host, port) && port == a.kubeletPort { + address = net.JoinHostPort(a.kubeletAddr, port) + } + return defaultDialer.DialContext(ctx, network, address) +} diff --git a/pkg/daemons/control/tunnel.go b/pkg/daemons/control/tunnel.go index 86c685318b3f..fba58aa4f77b 100644 --- a/pkg/daemons/control/tunnel.go +++ b/pkg/daemons/control/tunnel.go @@ -3,7 +3,6 @@ package control import ( "bufio" "context" - "fmt" "io" "net" "net/http" @@ -197,7 +196,6 @@ func (t *TunnelServer) dialBackend(ctx context.Context, addr string) (net.Conn, if err != nil { return nil, err } - loopback := t.config.Loopback(true) var nodeName string var toKubelet, useTunnel bool @@ -224,14 +222,17 @@ func (t *TunnelServer) dialBackend(ctx context.Context, addr string) (net.Conn, useTunnel = true } - // Always dial kubelet via the loopback address. - if toKubelet { - addr = fmt.Sprintf("%s:%s", loopback, port) - } - // If connecting to something hosted by the local node, don't tunnel if nodeName == t.config.ServerNodeName { useTunnel = false + if toKubelet { + // Dial local kubelet at the configured bind address + addr = net.JoinHostPort(t.config.BindAddress, port) + } + } else if toKubelet { + // Dial remote kubelet via the loopback address, the remotedialer client + // will ensure that it hits the right local address. + addr = net.JoinHostPort(t.config.Loopback(false), port) } if useTunnel {