From 83e0b0c19aaca7d413483b3a908c9dc3b4289203 Mon Sep 17 00:00:00 2001 From: Dmitriy Matrenichev Date: Mon, 12 Feb 2024 17:07:40 +0300 Subject: [PATCH] chore: adjust dns sockets settings Enable some TCP optimization, set minimal TTL, set socket reuse. Signed-off-by: Dmitriy Matrenichev --- go.mod | 2 +- go.sum | 4 +- .../controllers/network/dns_resolve_cache.go | 52 +++++++++--- internal/pkg/dns/dns.go | 85 +++++++++++++++++-- internal/pkg/dns/dns_test.go | 14 +-- 5 files changed, 128 insertions(+), 29 deletions(-) diff --git a/go.mod b/go.mod index b2ebad9351..dedcee51cf 100644 --- a/go.mod +++ b/go.mod @@ -111,7 +111,7 @@ require ( github.com/siderolabs/go-blockdevice v0.4.7 github.com/siderolabs/go-circular v0.1.0 github.com/siderolabs/go-cmd v0.1.1 - github.com/siderolabs/go-debug v0.2.3 + github.com/siderolabs/go-debug v0.3.0 github.com/siderolabs/go-kmsg v0.1.4 github.com/siderolabs/go-kubeconfig v0.1.0 github.com/siderolabs/go-kubernetes v0.2.8 diff --git a/go.sum b/go.sum index 74723fc873..ee00ed80e5 100644 --- a/go.sum +++ b/go.sum @@ -665,8 +665,8 @@ github.com/siderolabs/go-circular v0.1.0 h1:zpBJNUbCZSh0odZxA4Dcj0d3ShLLR2WxKW6h github.com/siderolabs/go-circular v0.1.0/go.mod h1:14XnLf/I3J0VjzTgmwWNGjp58/bdIi4zXppAEx8plfw= github.com/siderolabs/go-cmd v0.1.1 h1:nTouZUSxLeiiEe7hFexSVvaTsY/3O8k1s08BxPRrsps= github.com/siderolabs/go-cmd v0.1.1/go.mod h1:6hY0JG34LxEEwYE8aH2iIHkHX/ir12VRLqfwAf2yJIY= -github.com/siderolabs/go-debug v0.2.3 h1:O9luHW4P++gQqOKzMnUgGIiQGsg9QaQmi6qI0JfQHXI= -github.com/siderolabs/go-debug v0.2.3/go.mod h1:45N6ALRTyrbtKAzbehGwz3VoE5MZqEKd/xrv7dawR30= +github.com/siderolabs/go-debug v0.3.0 h1:C8t7jbac5Va2eYu9QRXXEGsy3Vz5xOEVo0TDwVJH268= +github.com/siderolabs/go-debug v0.3.0/go.mod h1:DonqzIQOm3+qof020meFwJ2gXI5Jv/x4Dj27FyUW4aE= github.com/siderolabs/go-kmsg v0.1.4 h1:RLAa90O9bWuhA3pXPAYAdrI+kzcqTshZASRA5yso/mo= github.com/siderolabs/go-kmsg v0.1.4/go.mod h1:BLkt2N2DHT0wsFMz32lMw6vNEZL90c8ZnBjpIUoBb/M= github.com/siderolabs/go-kubeconfig v0.1.0 h1:t/2oMWkLSdWHXglKPMz8ySXnx6ZjHckeGY79NaDcBTo= diff --git a/internal/app/machined/pkg/controllers/network/dns_resolve_cache.go b/internal/app/machined/pkg/controllers/network/dns_resolve_cache.go index 9e21d889ab..e6903e5460 100644 --- a/internal/app/machined/pkg/controllers/network/dns_resolve_cache.go +++ b/internal/app/machined/pkg/controllers/network/dns_resolve_cache.go @@ -103,27 +103,51 @@ func (ctrl *DNSResolveCacheController) runServer(originCtx context.Context, r co addr := ctrl.Addr ctx := originCtx - for _, opt := range []dns.ServerOptins{ + for _, opt := range []struct { + net string + addr string + srvOpts dns.ServerOptins + }{ { - Addr: addr, - Net: "udp", - Handler: cache, + net: "udp", + addr: addr, + srvOpts: dns.ServerOptins{ + Handler: cache, + }, }, { - Addr: addr, - Net: "tcp", - Handler: cache, - ReadTimeout: 3 * time.Second, - WriteTimeout: 5 * time.Second, - IdleTimeout: func() time.Duration { return 10 * time.Second }, - MaxTCPQueries: -1, + net: "tcp", + addr: addr, + srvOpts: dns.ServerOptins{ + Handler: cache, + ReadTimeout: 3 * time.Second, + WriteTimeout: 5 * time.Second, + IdleTimeout: func() time.Duration { return 10 * time.Second }, + MaxTCPQueries: -1, + }, }, } { - l := ctrl.Logger.With(zap.String("net", opt.Net)) + l := ctrl.Logger.With(zap.String("net", opt.net)) - runner := dns.NewRunner(dns.NewServer(opt), l) + if opt.net == "tcp" { + listener, err := dns.NewTCPListener(opt.addr) + if err != nil { + return fmt.Errorf("error creating tcp listener: %w", err) + } + + opt.srvOpts.Listener = listener + } else if opt.net == "udp" { + packetConn, err := dns.NewUDPPacketConn(opt.addr) + if err != nil { + return fmt.Errorf("error creating udp packet conn: %w", err) + } + + opt.srvOpts.PacketConn = packetConn + } + + runner := dns.NewRunner(dns.NewServer(opt.srvOpts), l) - err := ctrl.writeDNSStatus(ctx, r, opt.Net) + err := ctrl.writeDNSStatus(ctx, r, opt.net) if err != nil { return err } diff --git a/internal/pkg/dns/dns.go b/internal/pkg/dns/dns.go index f72928ed41..8737b0b18e 100644 --- a/internal/pkg/dns/dns.go +++ b/internal/pkg/dns/dns.go @@ -8,10 +8,13 @@ package dns import ( "context" "errors" + "fmt" "math/rand" + "net" "slices" "strings" "sync" + "syscall" "time" "github.com/coredns/coredns/plugin" @@ -20,13 +23,14 @@ import ( "github.com/coredns/coredns/request" "github.com/miekg/dns" "go.uber.org/zap" + "golang.org/x/sys/unix" "github.com/siderolabs/talos/internal/pkg/utils" ) // NewRunner creates a new Runner. func NewRunner(srv Server, logger *zap.Logger) *Runner { - r := utils.NewRunner(srv.ListenAndServe, srv.Shutdown, func(err error) bool { + r := utils.NewRunner(srv.ActivateAndServe, srv.Shutdown, func(err error) bool { // There a possible scenario where `Run` reached `ListenAndServe` and then yielded CPU time to another // goroutine and then `Stop` reached `Shutdown`. In that case `ListenAndServe` will actually start after // `Shutdown` and `Stop` method will forever block if we do not try again. @@ -44,7 +48,7 @@ type Runner struct { // Server is a dns server. type Server interface { - ListenAndServe() error + ActivateAndServe() error Shutdown() error } @@ -200,8 +204,8 @@ func (h *Handler) Stop() { h.SetProxy(nil) } // ServerOptins is a Server options. type ServerOptins struct { - Addr string - Net string + Listener net.Listener + PacketConn net.PacketConn Handler dns.Handler ReadTimeout time.Duration WriteTimeout time.Duration @@ -212,8 +216,8 @@ type ServerOptins struct { // NewServer creates a new Server. func NewServer(opts ServerOptins) Server { return &server{&dns.Server{ - Addr: opts.Addr, - Net: opts.Net, + Listener: opts.Listener, + PacketConn: opts.PacketConn, Handler: opts.Handler, ReadTimeout: opts.ReadTimeout, WriteTimeout: opts.WriteTimeout, @@ -223,3 +227,72 @@ func NewServer(opts ServerOptins) Server { } type server struct{ *dns.Server } + +// NewTCPListener creates a new TCP listener. +func NewTCPListener(addr string) (net.Listener, error) { + lc := net.ListenConfig{ + Control: makeControl(tcpOptions), + } + + return lc.Listen(context.Background(), "tcp", addr) +} + +// NewUDPPacketConn creates a new UDP packet connection. +func NewUDPPacketConn(addr string) (net.PacketConn, error) { + lc := net.ListenConfig{ + Control: makeControl(udpOptions), + } + + return lc.ListenPacket(context.Background(), "udp", addr) +} + +var ( + tcpOptions = []controlOptions{ + // this isn't really necessary, because currently if the process dies, OS dies with it + {unix.SOL_SOCKET, unix.SO_REUSEADDR, 1, "failed to set SO_REUSEADDR"}, + {unix.IPPROTO_IP, unix.IP_RECVTTL, 1, "failed to set IP_RECVTTL"}, + {unix.IPPROTO_TCP, unix.TCP_FASTOPEN, 5, "failed to set TCP_FASTOPEN"}, // tcp specific stuff from systemd + {unix.IPPROTO_TCP, unix.TCP_NODELAY, 1, "failed to set TCP_NODELAY"}, // tcp specific stuff from systemd + {unix.IPPROTO_IP, unix.IP_TTL, 1, "failed to set IP_TTL"}, + } + + udpOptions = []controlOptions{ + // this isn't really necessary, because currently if the process dies, OS dies with it + {unix.SOL_SOCKET, unix.SO_REUSEADDR, 1, "failed to set SO_REUSEADDR"}, + {unix.IPPROTO_IP, unix.IP_RECVTTL, 1, "failed to set IP_RECVTTL"}, + {unix.IPPROTO_IP, unix.IP_TTL, 1, "failed to set IP_TTL"}, + } +) + +type controlOptions struct { + level int + opt int + val int + errorMessage string +} + +func makeControl(opts []controlOptions) func(string, string, syscall.RawConn) error { + return func(_ string, _ string, c syscall.RawConn) error { + var resErr error + + err := c.Control(func(fd uintptr) { + for _, opt := range opts { + opErr := unix.SetsockoptInt(int(fd), opt.level, opt.opt, opt.val) + if opErr != nil { + resErr = fmt.Errorf(opt.errorMessage+": %w", opErr) + + return + } + } + }) + if err != nil { + return fmt.Errorf("failed in control call: %w", err) + } + + if resErr != nil { + return fmt.Errorf("failed to set socket options: %w", resErr) + } + + return nil + } +} diff --git a/internal/pkg/dns/dns_test.go b/internal/pkg/dns/dns_test.go index 024e63259b..647c56787b 100644 --- a/internal/pkg/dns/dns_test.go +++ b/internal/pkg/dns/dns_test.go @@ -120,10 +120,12 @@ func newServer(t *testing.T, nameservers ...string) (context.Context, func()) { handler.SetProxy(pxs) + pc, err := dns.NewUDPPacketConn(":10700") + require.NoError(t, err) + runner := dns.NewRunner(dns.NewServer(dns.ServerOptins{ - Addr: ":10700", - Net: "udp", - Handler: dns.NewCache(handler, l), + PacketConn: pc, + Handler: dns.NewCache(handler, l), }), l) return ctxutil.MonitorFn(context.Background(), runner.Run), runner.Stop @@ -145,7 +147,7 @@ func createQuery() *dnssrv.Msg { } } -func TestListenFailure(t *testing.T) { +func TestActivateFailure(t *testing.T) { // Ensure that we correctly handle an error inside [dns.Runner.Run]. l := zaptest.NewLogger(t) @@ -188,7 +190,7 @@ type testServer struct { var errFailed = errors.New("listen failure") -func (ts *testServer) ListenAndServe() error { return errFailed } +func (ts *testServer) ActivateAndServe() error { return errFailed } func (ts *testServer) Shutdown() error { ts.t.Fatal("should not be called") @@ -204,7 +206,7 @@ type runnerStopper struct { val atomic.Pointer[chan struct{}] } -func (rs *runnerStopper) ListenAndServe() error { +func (rs *runnerStopper) ActivateAndServe() error { ch := make(chan struct{}) if rs.val.Swap(&ch) != nil {