refactor(config): allow more configuration for upstreams

cmd/root_test.go

@@ -37,9 +37,10 @@ var _ = Describe("root command", func() {
 			DeferCleanup(tmpDir.Clean)
 
 			tmpFile = tmpDir.CreateStringFile("config",
-				"upstream:",
-				"  default:",
-				"    - 1.1.1.1",
+				"upstreams:",
+				"  groups:",
+				"    default:",
+				"      - 1.1.1.1",
 				"blocking:",
 				"  blackLists:",
 				"    ads:",

config/config.go

@@ -185,8 +185,7 @@ func (b *BootstrappedUpstreamConfig) UnmarshalYAML(unmarshal func(interface{}) e
 //
 //nolint:maligned
 type Config struct {
-	Upstream            ParallelBestConfig        `yaml:"upstream"`
-	UpstreamTimeout     Duration                  `yaml:"upstreamTimeout" default:"2s"`
+	Upstreams           UpstreamsConfig           `yaml:"upstreams"`
 	ConnectIPVersion    IPVersion                 `yaml:"connectIPVersion"`
 	CustomDNS           CustomDNSConfig           `yaml:"customDNS"`
 	Conditional         ConditionalUpstreamConfig `yaml:"conditional"`
@@ -212,15 +211,17 @@ type Config struct {
 
 	// Deprecated options
 	Deprecated struct {
-		DisableIPv6  *bool           `yaml:"disableIPv6"`
-		LogLevel     *log.Level      `yaml:"logLevel"`
-		LogFormat    *log.FormatType `yaml:"logFormat"`
-		LogPrivacy   *bool           `yaml:"logPrivacy"`
-		LogTimestamp *bool           `yaml:"logTimestamp"`
-		DNSPorts     *ListenConfig   `yaml:"port"`
-		HTTPPorts    *ListenConfig   `yaml:"httpPort"`
-		HTTPSPorts   *ListenConfig   `yaml:"httpsPort"`
-		TLSPorts     *ListenConfig   `yaml:"tlsPort"`
+		Upstream        *UpstreamGroups `yaml:"upstream"`
+		UpstreamTimeout *Duration       `yaml:"upstreamTimeout"`
+		DisableIPv6     *bool           `yaml:"disableIPv6"`
+		LogLevel        *log.Level      `yaml:"logLevel"`
+		LogFormat       *log.FormatType `yaml:"logFormat"`
+		LogPrivacy      *bool           `yaml:"logPrivacy"`
+		LogTimestamp    *bool           `yaml:"logTimestamp"`
+		DNSPorts        *ListenConfig   `yaml:"port"`
+		HTTPPorts       *ListenConfig   `yaml:"httpPort"`
+		HTTPSPorts      *ListenConfig   `yaml:"httpsPort"`
+		TLSPorts        *ListenConfig   `yaml:"tlsPort"`
 	} `yaml:",inline"`
 }
 
@@ -489,6 +490,8 @@ func unmarshalConfig(data []byte, cfg *Config) error {
 
 func (cfg *Config) migrate(logger *logrus.Entry) bool {
 	usesDepredOpts := Migrate(logger, "", cfg.Deprecated, map[string]Migrator{
+		"upstream":        Move(To("upstreams.groups", &cfg.Upstreams)),
+		"upstreamTimeout": Move(To("upstreams.timeout", &cfg.Upstreams)),
 		"disableIPv6": Apply(To("filtering.queryTypes", &cfg.Filtering), func(oldValue bool) {
 			if oldValue {
 				cfg.Filtering.QueryTypes.Insert(dns.Type(dns.TypeAAAA))

config/config_test.go

@@ -762,10 +762,10 @@ bootstrapDns:
 
 func defaultTestFileConfig() {
 	Expect(config.Ports.DNS).Should(Equal(ListenConfig{"55553", ":55554", "[::1]:55555"}))
-	Expect(config.Upstream.ExternalResolvers["default"]).Should(HaveLen(3))
-	Expect(config.Upstream.ExternalResolvers["default"][0].Host).Should(Equal("8.8.8.8"))
-	Expect(config.Upstream.ExternalResolvers["default"][1].Host).Should(Equal("8.8.4.4"))
-	Expect(config.Upstream.ExternalResolvers["default"][2].Host).Should(Equal("1.1.1.1"))
+	Expect(config.Upstreams.Groups["default"]).Should(HaveLen(3))
+	Expect(config.Upstreams.Groups["default"][0].Host).Should(Equal("8.8.8.8"))
+	Expect(config.Upstreams.Groups["default"][1].Host).Should(Equal("8.8.4.4"))
+	Expect(config.Upstreams.Groups["default"][2].Host).Should(Equal("1.1.1.1"))
 	Expect(config.CustomDNS.Mapping.HostIPs).Should(HaveLen(2))
 	Expect(config.CustomDNS.Mapping.HostIPs["my.duckdns.org"][0]).Should(Equal(net.ParseIP("192.168.178.3")))
 	Expect(config.CustomDNS.Mapping.HostIPs["multiple.ips"][0]).Should(Equal(net.ParseIP("192.168.178.3")))
@@ -797,11 +797,12 @@ func defaultTestFileConfig() {
 
 func writeConfigYml(tmpDir *helpertest.TmpFolder) *helpertest.TmpFile {
 	return tmpDir.CreateStringFile("config.yml",
-		"upstream:",
-		"  default:",
-		"    - tcp+udp:8.8.8.8",
-		"    - tcp+udp:8.8.4.4",
-		"    - 1.1.1.1",
+		"upstreams:",
+		"  groups:",
+		"    default:",
+		"      - tcp+udp:8.8.8.8",
+		"      - tcp+udp:8.8.4.4",
+		"      - 1.1.1.1",
 		"customDNS:",
 		"  mapping:",
 		"    my.duckdns.org: 192.168.178.3",
@@ -856,11 +857,12 @@ func writeConfigYml(tmpDir *helpertest.TmpFolder) *helpertest.TmpFile {
 
 func writeConfigDir(tmpDir *helpertest.TmpFolder) error {
 	f1 := tmpDir.CreateStringFile("config1.yaml",
-		"upstream:",
-		"  default:",
-		"    - tcp+udp:8.8.8.8",
-		"    - tcp+udp:8.8.4.4",
-		"    - 1.1.1.1",
+		"upstreams:",
+		"  groups:",
+		"    default:",
+		"      - tcp+udp:8.8.8.8",
+		"      - tcp+udp:8.8.4.4",
+		"      - 1.1.1.1",
 		"customDNS:",
 		"  mapping:",
 		"    my.duckdns.org: 192.168.178.3",

config/migration/migration.go

@@ -41,6 +41,10 @@ func Migrate(logger *logrus.Entry, optPrefix string, deprecated any, newOptions
 			panic(fmt.Errorf("deprecated option %s must be a pointer", oldName))
 		}
 
+		if field.Tag.Get("default") != "" {
+			panic(fmt.Errorf("deprecated option %s must not have a default", oldName))
+		}
+
 		if val.IsNil() {
 			// Deprecated option is not defined in the user's config
 			continue

config/upstreams.go

@@ -0,0 +1,35 @@
+package config
+
+import (
+	"github.com/sirupsen/logrus"
+)
+
+const UpstreamDefaultCfgName = "default"
+
+// UpstreamsConfig upstream servers configuration
+type UpstreamsConfig struct {
+	Timeout Duration       `yaml:"timeout" default:"2s"`
+	Groups  UpstreamGroups `yaml:"groups"`
+}
+
+type UpstreamGroups map[string][]Upstream
+
+// IsEnabled implements `config.Configurable`.
+func (c *UpstreamsConfig) IsEnabled() bool {
+	return len(c.Groups) != 0
+}
+
+// LogConfig implements `config.Configurable`.
+func (c *UpstreamsConfig) LogConfig(logger *logrus.Entry) {
+	logger.Info("timeout: ", c.Timeout)
+
+	logger.Info("groups:")
+
+	for name, upstreams := range c.Groups {
+		logger.Infof("  %s:", name)
+
+		for _, upstream := range upstreams {
+			logger.Infof("    - %s", upstream)
+		}
+	}
+}

config/parallel_best_test.go → config/upstreams_test.go

@@ -1,19 +1,22 @@
 package config
 
 import (
+	"time"
+
 	"github.com/creasty/defaults"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
 
 var _ = Describe("ParallelBestConfig", func() {
-	var cfg ParallelBestConfig
+	var cfg UpstreamsConfig
 
 	suiteBeforeEach()
 
 	BeforeEach(func() {
-		cfg = ParallelBestConfig{
-			ExternalResolvers: ParallelBestMapping{
+		cfg = UpstreamsConfig{
+			Timeout: Duration(5 * time.Second),
+			Groups: UpstreamGroups{
 				UpstreamDefaultCfgName: {
 					{Host: "host1"},
 					{Host: "host2"},
@@ -24,7 +27,7 @@ var _ = Describe("ParallelBestConfig", func() {
 
 	Describe("IsEnabled", func() {
 		It("should be false by default", func() {
-			cfg := ParallelBestConfig{}
+			cfg := UpstreamsConfig{}
 			Expect(defaults.Set(&cfg)).Should(Succeed())
 
 			Expect(cfg.IsEnabled()).Should(BeFalse())
@@ -38,7 +41,7 @@ var _ = Describe("ParallelBestConfig", func() {
 
 		When("disabled", func() {
 			It("should be false", func() {
-				cfg := ParallelBestConfig{}
+				cfg := UpstreamsConfig{}
 
 				Expect(cfg.IsEnabled()).Should(BeFalse())
 			})
@@ -50,7 +53,8 @@ var _ = Describe("ParallelBestConfig", func() {
 			cfg.LogConfig(logger)
 
 			Expect(hook.Calls).ShouldNot(BeEmpty())
-			Expect(hook.Messages).Should(ContainElement(ContainSubstring("upstream resolvers:")))
+			Expect(hook.Messages).Should(ContainElement(ContainSubstring("timeout:")))
+			Expect(hook.Messages).Should(ContainElement(ContainSubstring("groups:")))
 			Expect(hook.Messages).Should(ContainElement(ContainSubstring(":host2:")))
 		})
 	})

docs/config.yml

@@ -1,25 +1,25 @@
 # REVIEW: manual changelog entry
 
-upstream:
-  # these external DNS resolvers will be used. Blocky picks 2 random resolvers from the list for each query
-  # format for resolver: [net:]host:[port][/path]. net could be empty (default, shortcut for tcp+udp), tcp+udp, tcp, udp, tcp-tls or https (DoH). If port is empty, default port will be used (53 for udp and tcp, 853 for tcp-tls, 443 for https (Doh))
-  # this configuration is mandatory, please define at least one external DNS resolver
-  default:
-    # example for tcp+udp IPv4 server (https://digitalcourage.de/)
-    - 5.9.164.112
-    # Cloudflare
-    - 1.1.1.1
-    # example for DNS-over-TLS server (DoT)
-    - tcp-tls:fdns1.dismail.de:853
-    # example for DNS-over-HTTPS (DoH)
-    - https://dns.digitale-gesellschaft.ch/dns-query
-  # optional: use client name (with wildcard support: * - sequence of any characters, [0-9] - range)
-  # or single ip address / client subnet as CIDR notation
-  laptop*:
-    - 123.123.123.123
-
-# optional: timeout to query the upstream resolver. Default: 2s
-upstreamTimeout: 2s
+upstreams:
+  groups:
+    # these external DNS resolvers will be used. Blocky picks 2 random resolvers from the list for each query
+    # format for resolver: [net:]host:[port][/path]. net could be empty (default, shortcut for tcp+udp), tcp+udp, tcp, udp, tcp-tls or https (DoH). If port is empty, default port will be used (53 for udp and tcp, 853 for tcp-tls, 443 for https (Doh))
+    # this configuration is mandatory, please define at least one external DNS resolver
+    default:
+      # example for tcp+udp IPv4 server (https://digitalcourage.de/)
+      - 5.9.164.112
+      # Cloudflare
+      - 1.1.1.1
+      # example for DNS-over-TLS server (DoT)
+      - tcp-tls:fdns1.dismail.de:853
+      # example for DNS-over-HTTPS (DoH)
+      - https://dns.digitale-gesellschaft.ch/dns-query
+    # optional: use client name (with wildcard support: * - sequence of any characters, [0-9] - range)
+    # or single ip address / client subnet as CIDR notation
+    laptop*:
+      - 123.123.123.123
+  # optional: timeout to query the upstream resolver. Default: 2s
+  timeout: 2s
 
 # optional: If true, blocky will fail to start unless at least one upstream server per group is reachable. Default: false
 startVerifyUpstream: true

docs/configuration.md

@@ -68,7 +68,7 @@ All logging options are optional.
       privacy: true
     ```
 
-## Upstream configuration
+## Upstreams configuration
 
 To resolve a DNS query, blocky needs external public or private DNS resolvers. Blocky supports DNS resolvers with
 following network protocols (net part of the resolver URL):
@@ -107,17 +107,18 @@ CIDR notation.
 !!! example
 
     ```yaml
-    upstream:
-      default:
-        - 5.9.164.112
-        - 1.1.1.1
-        - tcp-tls:fdns1.dismail.de:853
-        - https://dns.digitale-gesellschaft.ch/dns-query
-      laptop*:
-        - 123.123.123.123
-      10.43.8.67/28:
-        - 1.1.1.1
-        - 9.9.9.9
+    upstreams:
+      groups:
+        default:
+          - 5.9.164.112
+          - 1.1.1.1
+          - tcp-tls:fdns1.dismail.de:853
+          - https://dns.digitale-gesellschaft.ch/dns-query
+        laptop*:
+          - 123.123.123.123
+        10.43.8.67/28:
+          - 1.1.1.1
+          - 9.9.9.9
     ```
 
 Use `123.123.123.123` as single upstream DNS resolver for client laptop-home,
@@ -133,16 +134,17 @@ public free DNS server you could use.
 ### Upstream lookup timeout
 
 Blocky will wait 2 seconds (default value) for the response from the external upstream DNS server. You can change this
-value by setting the `upstreamTimeout` configuration parameter (in **duration format**).
+value by setting the `timeout` configuration parameter (in **duration format**).
 
 !!! example
 
     ```yaml
-    upstream:
-      default:
-        - 46.182.19.48
-        - 80.241.218.68
-    upstreamTimeout: 5s
+    upstreams:
+      timeout: 5s
+      groups:
+        default:
+          - 46.182.19.48
+          - 80.241.218.68
     ```
 
 ## Bootstrap DNS configuration

e2e/basic_test.go

@@ -27,9 +27,10 @@ var _ = Describe("Basic functional tests", func() {
 		When("Minimal configuration is provided", func() {
 			BeforeEach(func() {
 				blocky, err = createBlockyContainer(tmpDir,
-					"upstream:",
-					"  default:",
-					"    - moka1",
+					"upstreams:",
+					"  groups:",
+					"    default:",
+					"      - moka1",
 				)
 
 				Expect(err).Should(Succeed())
@@ -58,9 +59,10 @@ var _ = Describe("Basic functional tests", func() {
 			When("'httpPort' is not defined", func() {
 				BeforeEach(func() {
 					blocky, err = createBlockyContainer(tmpDir,
-						"upstream:",
-						"  default:",
-						"    - moka1",
+						"upstreams:",
+						"  groups:",
+						"    default:",
+						"      - moka1",
 					)
 
 					Expect(err).Should(Succeed())
@@ -78,9 +80,10 @@ var _ = Describe("Basic functional tests", func() {
 			When("'httpPort' is defined", func() {
 				BeforeEach(func() {
 					blocky, err = createBlockyContainer(tmpDir,
-						"upstream:",
-						"  default:",
-						"    - moka1",
+						"upstreams:",
+						"  groups:",
+						"    default:",
+						"      - moka1",
 						"ports:",
 						"  http: 4000",
 					)