Support for multiple configuration files

README.md

@@ -65,7 +65,7 @@ Blocky is a DNS proxy and ad-blocker for the local network written in Go with fo
   * Various REST API endpoints
   * CLI tool
 
-- **Simple configuration** - single configuration file in YAML format
+- **Simple configuration** - single or multiple configuration files in YAML format
 
     * Simple to maintain
     * Simple to backup

cmd/root.go

@@ -40,7 +40,7 @@ Complete documentation is available at https://github.com/0xERR0R/blocky`,
 		SilenceUsage: true,
 	}
 
-	c.PersistentFlags().StringVarP(&configPath, "config", "c", defaultConfigPath, "path to config file")
+	c.PersistentFlags().StringVarP(&configPath, "config", "c", defaultConfigPath, "path to config file or folder")
 	c.PersistentFlags().StringVar(&apiHost, "apiHost", defaultHost, "host of blocky (API). Default overridden by config and CLI.") // nolint:lll
 	c.PersistentFlags().Uint16Var(&apiPort, "apiPort", defaultPort, "port of blocky (API). Default overridden by config and CLI.") // nolint:lll
 

config/config.go

@@ -7,6 +7,7 @@ import (
 	"io/ioutil"
 	"net"
 	"os"
+	"path/filepath"
 	"regexp"
 	"sort"
 	"strconv"
@@ -539,15 +540,14 @@ type FilteringConfig struct {
 // nolint:gochecknoglobals
 var config = &Config{}
 
-// LoadConfig creates new config from YAML file
+// LoadConfig creates new config from YAML file or a directory containing YAML files
 func LoadConfig(path string, mandatory bool) (*Config, error) {
 	cfg := Config{}
 	if err := defaults.Set(&cfg); err != nil {
 		return nil, fmt.Errorf("can't apply default values: %w", err)
 	}
 
-	data, err := ioutil.ReadFile(path)
-
+	fs, err := os.Stat(path)
 	if err != nil {
 		if errors.Is(err, os.ErrNotExist) && !mandatory {
 			// config file does not exist
@@ -557,7 +557,40 @@ func LoadConfig(path string, mandatory bool) (*Config, error) {
 			return config, nil
 		}
 
-		return nil, fmt.Errorf("can't read config file: %w", err)
+		return nil, fmt.Errorf("can't read config file(s): %w", err)
+	}
+
+	var data []byte
+
+	if fs.IsDir() { //nolint:nestif
+		err = filepath.WalkDir(path, func(filePath string, d os.DirEntry, err error) error {
+			if err != nil {
+				return err
+			}
+
+			if path == filePath {
+				return nil
+			}
+
+			fileData, err := os.ReadFile(filePath)
+			if err != nil {
+				return err
+			}
+
+			data = append(data, []byte("\n")...)
+			data = append(data, fileData...)
+
+			return nil
+		})
+
+		if err != nil {
+			return nil, fmt.Errorf("can't read config files: %w", err)
+		}
+	} else {
+		data, err = ioutil.ReadFile(path)
+		if err != nil {
+			return nil, fmt.Errorf("can't read config file: %w", err)
+		}
 	}
 
 	err = unmarshalConfig(data, &cfg)

config/config_test.go

@@ -24,36 +24,27 @@ var _ = Describe("Config", func() {
 				_, err = LoadConfig("config.yml", true)
 				Expect(err).Should(Succeed())
 
-				Expect(config.DNSPorts).Should(Equal(ListenConfig{"55553", ":55554", "[::1]:55555"}))
-				Expect(config.Upstream.ExternalResolvers["default"]).Should(HaveLen(3))
-				Expect(config.Upstream.ExternalResolvers["default"][0].Host).Should(Equal("8.8.8.8"))
-				Expect(config.Upstream.ExternalResolvers["default"][1].Host).Should(Equal("8.8.4.4"))
-				Expect(config.Upstream.ExternalResolvers["default"][2].Host).Should(Equal("1.1.1.1"))
-				Expect(config.CustomDNS.Mapping.HostIPs).Should(HaveLen(2))
-				Expect(config.CustomDNS.Mapping.HostIPs["my.duckdns.org"][0]).Should(Equal(net.ParseIP("192.168.178.3")))
-				Expect(config.CustomDNS.Mapping.HostIPs["multiple.ips"][0]).Should(Equal(net.ParseIP("192.168.178.3")))
-				Expect(config.CustomDNS.Mapping.HostIPs["multiple.ips"][1]).Should(Equal(net.ParseIP("192.168.178.4")))
-				Expect(config.CustomDNS.Mapping.HostIPs["multiple.ips"][2]).Should(Equal(
-					net.ParseIP("2001:0db8:85a3:08d3:1319:8a2e:0370:7344")))
-				Expect(config.Conditional.Mapping.Upstreams).Should(HaveLen(2))
-				Expect(config.Conditional.Mapping.Upstreams["fritz.box"]).Should(HaveLen(1))
-				Expect(config.Conditional.Mapping.Upstreams["multiple.resolvers"]).Should(HaveLen(2))
-				Expect(config.ClientLookup.Upstream.Host).Should(Equal("192.168.178.1"))
-				Expect(config.ClientLookup.SingleNameOrder).Should(Equal([]uint{2, 1}))
-				Expect(config.Blocking.BlackLists).Should(HaveLen(2))
-				Expect(config.Blocking.WhiteLists).Should(HaveLen(1))
-				Expect(config.Blocking.ClientGroupsBlock).Should(HaveLen(2))
-				Expect(config.Blocking.BlockTTL).Should(Equal(Duration(time.Minute)))
-				Expect(config.Blocking.RefreshPeriod).Should(Equal(Duration(2 * time.Hour)))
-				Expect(config.Filtering.QueryTypes).Should(HaveLen(2))
-
-				Expect(config.Caching.MaxCachingTime).Should(Equal(Duration(0)))
-				Expect(config.Caching.MinCachingTime).Should(Equal(Duration(0)))
-
-				Expect(config.DoHUserAgent).Should(Equal("testBlocky"))
-
-				Expect(GetConfig()).Should(Not(BeNil()))
+				defaultTestFileConfig()
+			})
+		})
+		When("Test file does not exist", func() {
+			It("should fail", func() {
+				_, err := LoadConfig("../testdata/config-does-not-exist.yaml", true)
+				Expect(err).Should(Not(Succeed()))
+			})
+		})
+		When("Multiple config files are used", func() {
+			It("should return a valid config struct", func() {
+				_, err := LoadConfig("../testdata/config/", true)
+				Expect(err).Should(Succeed())
 
+				defaultTestFileConfig()
+			})
+		})
+		When("Config folder does not exist", func() {
+			It("should fail", func() {
+				_, err := LoadConfig("../testdata/does-not-exist-config/", true)
+				Expect(err).Should(Not(Succeed()))
 			})
 		})
 		When("config file is malformed", func() {
@@ -618,3 +609,35 @@ bootstrapDns:
 		})
 	})
 })
+
+func defaultTestFileConfig() {
+	Expect(config.DNSPorts).Should(Equal(ListenConfig{"55553", ":55554", "[::1]:55555"}))
+	Expect(config.Upstream.ExternalResolvers["default"]).Should(HaveLen(3))
+	Expect(config.Upstream.ExternalResolvers["default"][0].Host).Should(Equal("8.8.8.8"))
+	Expect(config.Upstream.ExternalResolvers["default"][1].Host).Should(Equal("8.8.4.4"))
+	Expect(config.Upstream.ExternalResolvers["default"][2].Host).Should(Equal("1.1.1.1"))
+	Expect(config.CustomDNS.Mapping.HostIPs).Should(HaveLen(2))
+	Expect(config.CustomDNS.Mapping.HostIPs["my.duckdns.org"][0]).Should(Equal(net.ParseIP("192.168.178.3")))
+	Expect(config.CustomDNS.Mapping.HostIPs["multiple.ips"][0]).Should(Equal(net.ParseIP("192.168.178.3")))
+	Expect(config.CustomDNS.Mapping.HostIPs["multiple.ips"][1]).Should(Equal(net.ParseIP("192.168.178.4")))
+	Expect(config.CustomDNS.Mapping.HostIPs["multiple.ips"][2]).Should(Equal(
+		net.ParseIP("2001:0db8:85a3:08d3:1319:8a2e:0370:7344")))
+	Expect(config.Conditional.Mapping.Upstreams).Should(HaveLen(2))
+	Expect(config.Conditional.Mapping.Upstreams["fritz.box"]).Should(HaveLen(1))
+	Expect(config.Conditional.Mapping.Upstreams["multiple.resolvers"]).Should(HaveLen(2))
+	Expect(config.ClientLookup.Upstream.Host).Should(Equal("192.168.178.1"))
+	Expect(config.ClientLookup.SingleNameOrder).Should(Equal([]uint{2, 1}))
+	Expect(config.Blocking.BlackLists).Should(HaveLen(2))
+	Expect(config.Blocking.WhiteLists).Should(HaveLen(1))
+	Expect(config.Blocking.ClientGroupsBlock).Should(HaveLen(2))
+	Expect(config.Blocking.BlockTTL).Should(Equal(Duration(time.Minute)))
+	Expect(config.Blocking.RefreshPeriod).Should(Equal(Duration(2 * time.Hour)))
+	Expect(config.Filtering.QueryTypes).Should(HaveLen(2))
+
+	Expect(config.Caching.MaxCachingTime).Should(Equal(Duration(0)))
+	Expect(config.Caching.MinCachingTime).Should(Equal(Duration(0)))
+
+	Expect(config.DoHUserAgent).Should(Equal("testBlocky"))
+
+	Expect(GetConfig()).Should(Not(BeNil()))
+}

docs/installation.md

@@ -8,7 +8,7 @@ You can choose one of the following installation options:
 
 ## Prepare your configuration
 
-Blocky uses one YAML file as configuration. Create new `config.yaml` with your configuration (
+Blocky supports single or multiple YAML files as configuration. Create new `config.yaml` with your configuration (
 see [Configuration](configuration.md) for more details and all configuration options).
 
 Simple configuration file, which enables only basic features:
@@ -128,6 +128,16 @@ volumes:
       device: //NAS_HOSTNAME/blocky  
 ```
 
+#### Multiple configuration files
+
+For complex setups, splitting the configuration between multiple YAML files might be desired. In this case, folder containing YAML files is passed on startup, Blocky will join all the files.
+
+`./blocky --config ./config/`
+
+!!! warning
+
+    Blocky simply joins the multiple YAML files. If a directive (e.g. `upstream`) is repeated in multiple files, the configuration will not load and start will fail.
+
 ## Other installation types
 
 !!! warning

testdata/config/config1.yaml

@@ -0,0 +1,18 @@
+upstream:
+  default:
+    - tcp+udp:8.8.8.8
+    - tcp+udp:8.8.4.4
+    - 1.1.1.1
+customDNS:
+  mapping:
+    my.duckdns.org: 192.168.178.3
+    multiple.ips: 192.168.178.3,192.168.178.4,2001:0db8:85a3:08d3:1319:8a2e:0370:7344
+conditional:
+  mapping:
+    fritz.box: tcp+udp:192.168.178.1
+    multiple.resolvers: tcp+udp:192.168.178.1,tcp+udp:192.168.178.2
+filtering:
+  queryTypes:
+    - AAAA
+    - A
+

testdata/config/config2.yaml

@@ -0,0 +1,36 @@
+blocking:
+  blackLists:
+    ads:
+      - https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt
+      - https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts
+      - https://mirror1.malwaredomains.com/files/justdomains
+      - http://sysctl.org/cameleon/hosts
+      - https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist
+      - https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt
+    special:
+      - https://hosts-file.net/ad_servers.txt
+  whiteLists:
+    ads:
+      - whitelist.txt
+  clientGroupsBlock:
+    default:
+      - ads
+      - special
+    Laptop-D.fritz.box:
+      - ads
+  blockTTL: 1m
+  # without unit -> use minutes
+  refreshPeriod: 120
+clientLookup:
+  upstream: 192.168.178.1
+  singleNameOrder:
+      - 2
+      - 1
+
+queryLog:
+  type: csv-client
+  target: /opt/log
+
+port: 55553,:55554,[::1]:55555
+logLevel: debug
+dohUserAgent: testBlocky