Pull request 1891: 5902-bootstrap-hosts

Merge in DNS/adguard-home from 5902-bootstrap-hosts to master

Updates #5902.

Squashed commit of the following:

commit fcc65d3
Merge: 0c336af 1fd6cf1
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Fri Jun 30 12:29:06 2023 +0300

    Merge branch 'master' into 5902-bootstrap-hosts

commit 0c336af
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Thu Jun 29 15:40:28 2023 +0300

    all: imp & simplify

commit 45aae90
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Wed Jun 28 20:24:43 2023 +0300

    all: imp code, docs

commit e3dbb5b
Merge: a33a8e9 2069edd
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Wed Jun 28 18:27:36 2023 +0300

    Merge branch 'master' into 5902-bootstrap-hosts

commit a33a8e9
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Wed Jun 28 13:27:11 2023 +0300

    aghos: add type check

commit 781a3a2
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Wed Jun 28 13:09:37 2023 +0300

    all: log changes

commit 4575368
Merge: 636c440 cf7c12c
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Wed Jun 28 13:08:11 2023 +0300

    Merge branch 'master' into 5902-bootstrap-hosts

commit 636c440
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Wed Jun 28 13:06:32 2023 +0300

    all: imp tests

commit 0eff7a7
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Mon Jun 26 18:40:22 2023 +0300

    dnsforward: imp code

commit 7489a30
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Mon Jun 26 17:04:10 2023 +0300

    all: resolve upstreams with hosts

CHANGELOG.md

@@ -143,6 +143,8 @@ In this release, the schema version has changed from 20 to 23.
 
 ### Fixed
 
+- Using of `/etc/hosts` file to resolve the hostnames of upstream DNS servers
+  ([#5902]).
 - Excessive error logging when using DNS-over-QUIC ([#5285]).
 - Cannot set `bind_host` in AdGuardHome.yaml (docker version)([#4231], [#4235]).
 - The blocklists can now be deleted properly ([#5700]).
@@ -157,6 +159,7 @@ In this release, the schema version has changed from 20 to 23.
 [#4235]: https://github.com/AdguardTeam/AdGuardHome/pull/4235
 [#5285]: https://github.com/AdguardTeam/AdGuardHome/issues/5285
 [#5700]: https://github.com/AdguardTeam/AdGuardHome/issues/5700
+[#5902]: https://github.com/AdguardTeam/AdGuardHome/issues/5902
 [#5910]: https://github.com/AdguardTeam/AdGuardHome/issues/5910
 [#5913]: https://github.com/AdguardTeam/AdGuardHome/issues/5913
 

internal/aghnet/hostscontainer.go

@@ -56,15 +56,20 @@ func (rm *requestMatcher) MatchRequest(
 ) (res *urlfilter.DNSResult, ok bool) {
 	switch req.DNSType {
 	case dns.TypeA, dns.TypeAAAA, dns.TypePTR:
-		log.Debug("%s: handling the request for %s", hostsContainerPrefix, req.Hostname)
+		log.Debug(
+			"%s: handling %s request for %s",
+			hostsContainerPrefix,
+			dns.Type(req.DNSType),
+			req.Hostname,
+		)
+
+		rm.stateLock.RLock()
+		defer rm.stateLock.RUnlock()
+
+		return rm.engine.MatchRequest(req)
 	default:
 		return nil, false
 	}
-
-	rm.stateLock.RLock()
-	defer rm.stateLock.RUnlock()
-
-	return rm.engine.MatchRequest(req)
 }
 
 // Translate returns the source hosts-syntax rule for the generated dnsrewrite
@@ -96,6 +101,8 @@ const hostsContainerPrefix = "hosts container"
 
 // HostsContainer stores the relevant hosts database provided by the OS and
 // processes both A/AAAA and PTR DNS requests for those.
+//
+// TODO(e.burkov):  Improve API and move to golibs.
 type HostsContainer struct {
 	// requestMatcher matches the requests and translates the rules.  It's
 	// embedded to implement MatchRequest and Translate for *HostsContainer.

internal/dnsforward/config.go

@@ -15,7 +15,6 @@ import (
 	"github.com/AdguardTeam/AdGuardHome/internal/aghtls"
 	"github.com/AdguardTeam/AdGuardHome/internal/filtering"
 	"github.com/AdguardTeam/dnsproxy/proxy"
-	"github.com/AdguardTeam/dnsproxy/upstream"
 	"github.com/AdguardTeam/golibs/errors"
 	"github.com/AdguardTeam/golibs/log"
 	"github.com/AdguardTeam/golibs/netutil"
@@ -436,102 +435,6 @@ func (s *Server) initDefaultSettings() {
 	}
 }
 
-// UpstreamHTTPVersions returns the HTTP versions for upstream configuration
-// depending on configuration.
-func UpstreamHTTPVersions(http3 bool) (v []upstream.HTTPVersion) {
-	if !http3 {
-		return upstream.DefaultHTTPVersions
-	}
-
-	return []upstream.HTTPVersion{
-		upstream.HTTPVersion3,
-		upstream.HTTPVersion2,
-		upstream.HTTPVersion11,
-	}
-}
-
-// prepareUpstreamSettings - prepares upstream DNS server settings
-func (s *Server) prepareUpstreamSettings() error {
-	// We're setting a customized set of RootCAs.  The reason is that Go default
-	// mechanism of loading TLS roots does not always work properly on some
-	// routers so we're loading roots manually and pass it here.
-	//
-	// See [aghtls.SystemRootCAs].
-	upstream.RootCAs = s.conf.TLSv12Roots
-	upstream.CipherSuites = s.conf.TLSCiphers
-
-	// Load upstreams either from the file, or from the settings
-	var upstreams []string
-	if s.conf.UpstreamDNSFileName != "" {
-		data, err := os.ReadFile(s.conf.UpstreamDNSFileName)
-		if err != nil {
-			return fmt.Errorf("reading upstream from file: %w", err)
-		}
-
-		upstreams = stringutil.SplitTrimmed(string(data), "\n")
-
-		log.Debug("dns: using %d upstream servers from file %s", len(upstreams), s.conf.UpstreamDNSFileName)
-	} else {
-		upstreams = s.conf.UpstreamDNS
-	}
-
-	httpVersions := UpstreamHTTPVersions(s.conf.UseHTTP3Upstreams)
-	upstreams = stringutil.FilterOut(upstreams, IsCommentOrEmpty)
-	upstreamConfig, err := proxy.ParseUpstreamsConfig(
-		upstreams,
-		&upstream.Options{
-			Bootstrap:    s.conf.BootstrapDNS,
-			Timeout:      s.conf.UpstreamTimeout,
-			HTTPVersions: httpVersions,
-			PreferIPv6:   s.conf.BootstrapPreferIPv6,
-		},
-	)
-	if err != nil {
-		return fmt.Errorf("parsing upstream config: %w", err)
-	}
-
-	if len(upstreamConfig.Upstreams) == 0 {
-		log.Info("warning: no default upstream servers specified, using %v", defaultDNS)
-		var uc *proxy.UpstreamConfig
-		uc, err = proxy.ParseUpstreamsConfig(
-			defaultDNS,
-			&upstream.Options{
-				Bootstrap:    s.conf.BootstrapDNS,
-				Timeout:      s.conf.UpstreamTimeout,
-				HTTPVersions: httpVersions,
-				PreferIPv6:   s.conf.BootstrapPreferIPv6,
-			},
-		)
-		if err != nil {
-			return fmt.Errorf("parsing default upstreams: %w", err)
-		}
-
-		upstreamConfig.Upstreams = uc.Upstreams
-	}
-
-	s.conf.UpstreamConfig = upstreamConfig
-
-	return nil
-}
-
-// setProxyUpstreamMode sets the upstream mode and related settings in conf
-// based on provided parameters.
-func setProxyUpstreamMode(
-	conf *proxy.Config,
-	allServers bool,
-	fastestAddr bool,
-	fastestTimeout time.Duration,
-) {
-	if allServers {
-		conf.UpstreamMode = proxy.UModeParallel
-	} else if fastestAddr {
-		conf.UpstreamMode = proxy.UModeFastestAddr
-		conf.FastestPingTimeout = fastestTimeout
-	} else {
-		conf.UpstreamMode = proxy.UModeLoadBalance
-	}
-}
-
 // prepareIpsetListSettings reads and prepares the ipset configuration either
 // from a file or from the data in the configuration file.
 func (s *Server) prepareIpsetListSettings() (err error) {

internal/dnsforward/dnsforward.go

@@ -466,19 +466,15 @@ func (s *Server) setupResolvers(localAddrs []string) (err error) {
 
 	log.Debug("dnsforward: upstreams to resolve ptr for local addresses: %v", localAddrs)
 
-	var upsConfig *proxy.UpstreamConfig
-	upsConfig, err = proxy.ParseUpstreamsConfig(
-		localAddrs,
-		&upstream.Options{
-			Bootstrap: bootstraps,
-			Timeout:   defaultLocalTimeout,
-			// TODO(e.burkov): Should we verify server's certificates?
-
-			PreferIPv6: s.conf.BootstrapPreferIPv6,
-		},
-	)
+	upsConfig, err := s.prepareUpstreamConfig(localAddrs, nil, &upstream.Options{
+		Bootstrap: bootstraps,
+		Timeout:   defaultLocalTimeout,
+		// TODO(e.burkov): Should we verify server's certificates?
+
+		PreferIPv6: s.conf.BootstrapPreferIPv6,
+	})
 	if err != nil {
-		return fmt.Errorf("parsing upstreams: %w", err)
+		return fmt.Errorf("parsing private upstreams: %w", err)
 	}
 
 	s.localResolvers = &proxy.Proxy{
@@ -510,7 +506,8 @@ func (s *Server) Prepare(conf *ServerConfig) (err error) {
 
 	err = s.prepareUpstreamSettings()
 	if err != nil {
-		return fmt.Errorf("preparing upstream settings: %w", err)
+		// Don't wrap the error, because it's informative enough as is.
+		return err
 	}
 
 	var proxyConfig proxy.Config

internal/dnsforward/http.go

@@ -633,61 +633,70 @@ func (err domainSpecificTestError) Error() (msg string) {
 	return fmt.Sprintf("WARNING: %s", err.error)
 }
 
-// checkDNS checks the upstream server defined by upstreamConfigStr using
-// healthCheck for actually exchange messages.  It uses bootstrap to resolve the
-// upstream's address.
-func checkDNS(
-	upstreamConfigStr string,
-	bootstrap []string,
-	bootstrapPrefIPv6 bool,
-	timeout time.Duration,
-	healthCheck healthCheckFunc,
-) (err error) {
-	if IsCommentOrEmpty(upstreamConfigStr) {
-		return nil
-	}
-
+// parseUpstreamLine parses line and creates the [upstream.Upstream] using opts
+// and information from [s.dnsFilter.EtcHosts].  It returns an error if the line
+// is not a valid upstream line, see [upstream.AddressToUpstream].  It's a
+// caller's responsibility to close u.
+func (s *Server) parseUpstreamLine(
+	line string,
+	opts *upstream.Options,
+) (u upstream.Upstream, specific bool, err error) {
 	// Separate upstream from domains list.
-	upstreamAddr, domains, err := separateUpstream(upstreamConfigStr)
+	upstreamAddr, domains, err := separateUpstream(line)
 	if err != nil {
-		return fmt.Errorf("wrong upstream format: %w", err)
+		return nil, false, fmt.Errorf("wrong upstream format: %w", err)
 	}
 
+	specific = len(domains) > 0
+
 	useDefault, err := validateUpstream(upstreamAddr, domains)
 	if err != nil {
-		return fmt.Errorf("wrong upstream format: %w", err)
+		return nil, specific, fmt.Errorf("wrong upstream format: %w", err)
 	} else if useDefault {
-		return nil
-	}
-
-	if len(bootstrap) == 0 {
-		bootstrap = defaultBootstrap
+		return nil, specific, nil
 	}
 
 	log.Debug("dnsforward: checking if upstream %q works", upstreamAddr)
 
-	u, err := upstream.AddressToUpstream(upstreamAddr, &upstream.Options{
-		Bootstrap:  bootstrap,
-		Timeout:    timeout,
-		PreferIPv6: bootstrapPrefIPv6,
-	})
+	opts = &upstream.Options{
+		Bootstrap:  opts.Bootstrap,
+		Timeout:    opts.Timeout,
+		PreferIPv6: opts.PreferIPv6,
+	}
+
+	if s.dnsFilter != nil && s.dnsFilter.EtcHosts != nil {
+		resolved := s.resolveUpstreamHost(extractUpstreamHost(upstreamAddr))
+		sortNetIPAddrs(resolved, opts.PreferIPv6)
+		opts.ServerIPAddrs = resolved
+	}
+	u, err = upstream.AddressToUpstream(upstreamAddr, opts)
 	if err != nil {
-		return fmt.Errorf("failed to choose upstream for %q: %w", upstreamAddr, err)
+		return nil, specific, fmt.Errorf("creating upstream for %q: %w", upstreamAddr, err)
 	}
-	defer func() { err = errors.WithDeferred(err, u.Close()) }()
 
-	if err = healthCheck(u); err != nil {
-		err = fmt.Errorf("upstream %q fails to exchange: %w", upstreamAddr, err)
-		if domains != nil {
-			return domainSpecificTestError{error: err}
+	return u, specific, nil
+}
+
+func (s *Server) checkDNS(line string, opts *upstream.Options, check healthCheckFunc) (err error) {
+	if IsCommentOrEmpty(line) {
+		return nil
+	}
+
+	var u upstream.Upstream
+	var specific bool
+	defer func() {
+		if err != nil && specific {
+			err = domainSpecificTestError{error: err}
 		}
+	}()
 
+	u, specific, err = s.parseUpstreamLine(line, opts)
+	if err != nil || u == nil {
 		return err
 	}
+	defer func() { err = errors.WithDeferred(err, u.Close()) }()
 
-	log.Debug("dnsforward: upstream %q is ok", upstreamAddr)
-
-	return nil
+	return check(u)
 }
 
 func (s *Server) handleTestUpstreamDNS(w http.ResponseWriter, r *http.Request) {
@@ -699,47 +708,54 @@ func (s *Server) handleTestUpstreamDNS(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	result := map[string]string{}
-	bootstraps := req.BootstrapDNS
-	bootstrapPrefIPv6 := s.conf.BootstrapPreferIPv6
-	timeout := s.conf.UpstreamTimeout
+	opts := &upstream.Options{
+		Bootstrap:  req.BootstrapDNS,
+		Timeout:    s.conf.UpstreamTimeout,
+		PreferIPv6: s.conf.BootstrapPreferIPv6,
+	}
+	if len(opts.Bootstrap) == 0 {
+		opts.Bootstrap = defaultBootstrap
+	}
 
 	type upsCheckResult = struct {
-		res  string
+		err  error
 		host string
 	}
 
+	req.Upstreams = stringutil.FilterOut(req.Upstreams, IsCommentOrEmpty)
+	req.PrivateUpstreams = stringutil.FilterOut(req.PrivateUpstreams, IsCommentOrEmpty)
+
 	upsNum := len(req.Upstreams) + len(req.PrivateUpstreams)
+	result := make(map[string]string, upsNum)
 	resCh := make(chan upsCheckResult, upsNum)
 
-	checkUps := func(ups string, healthCheck healthCheckFunc) {
-		res := upsCheckResult{
-			host: ups,
-		}
-		defer func() { resCh <- res }()
-
-		checkErr := checkDNS(ups, bootstraps, bootstrapPrefIPv6, timeout, healthCheck)
-		if checkErr != nil {
-			res.res = checkErr.Error()
-		} else {
-			res.res = "OK"
-		}
-	}
-
 	for _, ups := range req.Upstreams {
-		go checkUps(ups, checkDNSUpstreamExc)
+		go func(ups string) {
+			resCh <- upsCheckResult{
+				host: ups,
+				err:  s.checkDNS(ups, opts, checkDNSUpstreamExc),
+			}
+		}(ups)
 	}
 	for _, ups := range req.PrivateUpstreams {
-		go checkUps(ups, checkPrivateUpstreamExc)
+		go func(ups string) {
+			resCh <- upsCheckResult{
+				host: ups,
+				err:  s.checkDNS(ups, opts, checkPrivateUpstreamExc),
+			}
+		}(ups)
 	}
 
 	for i := 0; i < upsNum; i++ {
-		pair := <-resCh
 		// TODO(e.burkov):  The upstreams used for both common and private
 		// resolving should be reported separately.
-		result[pair.host] = pair.res
+		pair := <-resCh
+		if pair.err != nil {
+			result[pair.host] = pair.err.Error()
+		} else {
+			result[pair.host] = "OK"
+		}
 	}
-	close(resCh)
 
 	_ = aghhttp.WriteJSONResponse(w, r, result)
 }