Pull request 2114: 6480 upd proxy

Updates AdguardTeam#6480.

Squashed commit of the following:

commit 03d73fe
Merge: b6f9bd5 1511fab
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Mon Dec 25 14:50:08 2023 +0300

    Merge branch 'master' into 6480-upd-proxy

commit b6f9bd5
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Tue Dec 19 19:31:14 2023 +0300

    dnsforward: imp code, docs

commit db0356c
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Tue Dec 19 17:21:53 2023 +0300

    all: log changes

commit 0cbb0e8
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Tue Dec 19 16:26:34 2023 +0300

    all: upd proxy

CHANGELOG.md

@@ -58,6 +58,7 @@ NOTE: Add new changes BELOW THIS COMMENT.
 
 ### Fixed
 
+- Load balancing algorithm stuck on a single server ([#6480]).
 - Statistics for 7 days displayed as 168 hours on the dashboard.
 - Pre-filling the Edit static lease window with data ([#6534]).
 - Names defined in the `/etc/hosts` for a single address family wrongly
@@ -66,6 +67,7 @@ NOTE: Add new changes BELOW THIS COMMENT.
   work on iOS ([#6352]).
 
 [#6352]: https://github.com/AdguardTeam/AdGuardHome/issues/6352
+[#6480]: https://github.com/AdguardTeam/AdGuardHome/issues/6480
 [#6534]: https://github.com/AdguardTeam/AdGuardHome/issues/6534
 [#6541]: https://github.com/AdguardTeam/AdGuardHome/issues/6541
 [#6545]: https://github.com/AdguardTeam/AdGuardHome/issues/6545

go.mod

@@ -3,8 +3,8 @@ module github.com/AdguardTeam/AdGuardHome
 go 1.20
 
 require (
-	github.com/AdguardTeam/dnsproxy v0.60.1
-	github.com/AdguardTeam/golibs v0.18.0
+	github.com/AdguardTeam/dnsproxy v0.61.0
+	github.com/AdguardTeam/golibs v0.18.1
 	github.com/AdguardTeam/urlfilter v0.17.3
 	github.com/NYTimes/gziphandler v1.1.1
 	github.com/ameshkov/dnscrypt/v2 v2.2.7
@@ -64,4 +64,6 @@ require (
 	golang.org/x/sync v0.5.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/tools v0.16.0 // indirect
+	gonum.org/v1/gonum v0.14.0 // indirect
 )
+

go.sum

@@ -1,7 +1,7 @@
-github.com/AdguardTeam/dnsproxy v0.60.1 h1:YveGe7UZLaAiePkaV3orkc0IIfPX9vi/qQDIFdeO//A=
-github.com/AdguardTeam/dnsproxy v0.60.1/go.mod h1:B7FvvTFQZBfey1cJXQo732EyCLX6xj4JqrciCawATzg=
-github.com/AdguardTeam/golibs v0.18.0 h1:ckS2YK7t2Ub6UkXl0fnreVaM15Zb07Hh1gmFqttjpWg=
-github.com/AdguardTeam/golibs v0.18.0/go.mod h1:DKhCIXHcUYtBhU8ibTLKh1paUL96n5zhQBlx763sj+U=
+github.com/AdguardTeam/dnsproxy v0.61.0 h1:A5tmOpPw9c1kw5L4RKrOPzscRZuzpLaikwXLDsibQnY=
+github.com/AdguardTeam/dnsproxy v0.61.0/go.mod h1:IdmXdkpc+m+S2EajJkVZDZm//yQ4mQm2FCOugQpc/N8=
+github.com/AdguardTeam/golibs v0.18.1 h1:6u0fvrIj2qjUsRdbIGJ9AR0g5QRSWdKIo/DYl3tp5aM=
+github.com/AdguardTeam/golibs v0.18.1/go.mod h1:DKhCIXHcUYtBhU8ibTLKh1paUL96n5zhQBlx763sj+U=
 github.com/AdguardTeam/urlfilter v0.17.3 h1:fg/ObbnO0Cv6aw0tW6N/ETDMhhNvmcUUOZ7HlmKC3rw=
 github.com/AdguardTeam/urlfilter v0.17.3/go.mod h1:Jru7jFfeH2CoDf150uDs+rRYcZBzHHBz05r9REyDKyE=
 github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I=
@@ -164,6 +164,8 @@ golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM=
 golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gonum.org/v1/gonum v0.14.0 h1:2NiG67LD1tEH0D7kM+ps2V+fXmsAnpUeec7n8tcr4S0=
+gonum.org/v1/gonum v0.14.0/go.mod h1:AoWeoz0becf9QMWtE8iWXNXc27fK4fNeHNf/oMejGfU=
 google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=

internal/dnsforward/config.go

@@ -150,7 +150,7 @@ type Config struct {
 
 	// MaxGoroutines is the max number of parallel goroutines for processing
 	// incoming requests.
-	MaxGoroutines uint32 `yaml:"max_goroutines"`
+	MaxGoroutines uint `yaml:"max_goroutines"`
 
 	// HandleDDR, if true, handle DDR requests
 	HandleDDR bool `yaml:"handle_ddr"`
@@ -319,7 +319,7 @@ func (s *Server) newProxyConfig() (conf *proxy.Config, err error) {
 		RequestHandler:         s.handleDNSRequest,
 		HTTPSServerName:        aghhttp.UserAgent(),
 		EnableEDNSClientSubnet: srvConf.EDNSClientSubnet.Enabled,
-		MaxGoroutines:          int(srvConf.MaxGoroutines),
+		MaxGoroutines:          srvConf.MaxGoroutines,
 		UseDNS64:               srvConf.UseDNS64,
 		DNS64Prefs:             srvConf.DNS64Prefixes,
 	}

internal/dnsforward/dnsforward.go

@@ -81,6 +81,7 @@ type DHCP interface {
 	Enabled() (ok bool)
 }
 
+// SystemResolvers is an interface for accessing the OS-provided resolvers.
 type SystemResolvers interface {
 	// Addrs returns the list of system resolvers' addresses.
 	Addrs() (addrs []netip.AddrPort)
@@ -469,13 +470,15 @@ func (s *Server) startLocked() error {
 	return err
 }
 
-// setupLocalResolvers initializes the resolvers for local addresses.  It
-// assumes s.serverLock is locked or the Server not running.
-func (s *Server) setupLocalResolvers(boot upstream.Resolver) (err error) {
+// prepareLocalResolvers initializes the local upstreams configuration using
+// boot as bootstrap.  It assumes that s.serverLock is locked or s not running.
+func (s *Server) prepareLocalResolvers(
+	boot upstream.Resolver,
+) (uc *proxy.UpstreamConfig, err error) {
 	set, err := s.conf.ourAddrsSet()
 	if err != nil {
 		// Don't wrap the error because it's informative enough as is.
-		return err
+		return nil, err
 	}
 
 	resolvers := s.conf.LocalPTRResolvers
@@ -492,29 +495,46 @@ func (s *Server) setupLocalResolvers(boot upstream.Resolver) (err error) {
 
 	log.Debug("dnsforward: upstreams to resolve ptr for local addresses: %v", resolvers)
 
-	uc, err := s.prepareUpstreamConfig(resolvers, nil, &upstream.Options{
+	uc, err = s.prepareUpstreamConfig(resolvers, nil, &upstream.Options{
 		Bootstrap: boot,
 		Timeout:   defaultLocalTimeout,
 		// TODO(e.burkov): Should we verify server's certificates?
 		PreferIPv6: s.conf.BootstrapPreferIPv6,
 	})
 	if err != nil {
-		return fmt.Errorf("preparing private upstreams: %w", err)
+		return nil, fmt.Errorf("preparing private upstreams: %w", err)
 	}
 
 	if confNeedsFiltering {
 		err = filterOutAddrs(uc, set)
 		if err != nil {
-			return fmt.Errorf("filtering private upstreams: %w", err)
+			return nil, fmt.Errorf("filtering private upstreams: %w", err)
 		}
 	}
 
+	return uc, nil
+}
+
+// setupLocalResolvers initializes and sets the resolvers for local addresses.
+// It assumes s.serverLock is locked or s not running.
+func (s *Server) setupLocalResolvers(boot upstream.Resolver) (err error) {
+	uc, err := s.prepareLocalResolvers(boot)
+	if err != nil {
+		// Don't wrap the error because it's informative enough as is.
+		return err
+	}
+
 	s.localResolvers = &proxy.Proxy{
 		Config: proxy.Config{
 			UpstreamConfig: uc,
 		},
 	}
 
+	err = s.localResolvers.Init()
+	if err != nil {
+		return fmt.Errorf("initializing proxy: %w", err)
+	}
+
 	// TODO(e.burkov):  Should we also consider the DNS64 usage?
 	if s.conf.UsePrivateRDNS &&
 		// Only set the upstream config if there are any upstreams.  It's safe
@@ -700,7 +720,7 @@ func (s *Server) prepareInternalProxy() (err error) {
 		CacheEnabled:   true,
 		CacheSizeBytes: 4096,
 		UpstreamConfig: srvConf.UpstreamConfig,
-		MaxGoroutines:  int(s.conf.MaxGoroutines),
+		MaxGoroutines:  s.conf.MaxGoroutines,
 	}
 
 	err = setProxyUpstreamMode(conf, srvConf.UpstreamMode, srvConf.FastestTimeout.Duration)

internal/dnsforward/dnsforward_test.go

@@ -1547,9 +1547,9 @@ func TestServer_Exchange(t *testing.T) {
 			},
 		},
 	}
-
 	srv.conf.UsePrivateRDNS = true
 	srv.privateNets = netutil.SubnetSetFunc(netutil.IsLocallyServed)
+	require.NoError(t, srv.internalProxy.Init())
 
 	testCases := []struct {
 		req         netip.Addr
@@ -1625,6 +1625,7 @@ func TestServer_Exchange(t *testing.T) {
 		srv.localResolvers = &proxy.Proxy{
 			Config: pcfg,
 		}
+		require.NoError(t, srv.localResolvers.Init())
 
 		t.Run(tc.name, func(t *testing.T) {
 			host, ttl, eerr := srv.Exchange(tc.req)

internal/dnsforward/process.go

@@ -639,8 +639,7 @@ func (s *Server) processLocalPTR(dctx *dnsContext) (rc resultCode) {
 			// Generate the server failure if the private upstream configuration
 			// is empty.
 			//
-			// TODO(e.burkov):  Get rid of this crutch once the local resolvers
-			// logic is moved to the dnsproxy completely.
+			// This is a crutch, see TODO at [Server.localResolvers].
 			if errors.Is(err, upstream.ErrNoUpstreams) {
 				pctx.Res = s.genServerFailure(pctx.Req)