From 286d35bfb8a9b0db2901e49c5edff65fc965ea10 Mon Sep 17 00:00:00 2001 From: Ainar Garipov Date: Mon, 21 Dec 2020 17:48:07 +0300 Subject: [PATCH] Pull request: all: add $dnsrewrite handling Merge in DNS/adguard-home from 2102-dnsrewrite to master Updates #2102. Squashed commit of the following: commit 8490fc18179d38c4b162ff9b257fea1f8535afbd Merge: d9448ddca e7f7799b3 Author: Ainar Garipov Date: Mon Dec 21 16:44:00 2020 +0300 Merge branch 'master' into 2102-dnsrewrite commit d9448ddca6d4ef3635d767e3e496e44c35d3fc6e Author: Ainar Garipov Date: Mon Dec 21 15:44:54 2020 +0300 querylog: support dnsrewrite rules commit 40aa5d30acddf29fb90d249d8806941c6e1915a4 Author: Ainar Garipov Date: Fri Dec 18 19:27:40 2020 +0300 all: improve documentation commit f776a0cd63b1640ba1e5210d9301e2a2801fd824 Author: Ainar Garipov Date: Fri Dec 18 19:09:08 2020 +0300 dnsfilter: prevent panics, improve docs commit e14073b7500d9ed827a151c5b8fb863c980c10e8 Author: Ainar Garipov Date: Fri Dec 4 15:51:02 2020 +0300 all: add $dnsrewrite handling --- .githooks/pre-commit | 4 +- CHANGELOG.md | 1 + Makefile | 4 +- README.md | 4 +- go.mod | 2 +- go.sum | 4 +- internal/dnsfilter/dnsfilter.go | 150 +++++++------ internal/dnsfilter/dnsfilter_test.go | 63 +++--- internal/dnsfilter/dnsrewrite.go | 80 +++++++ internal/dnsfilter/dnsrewrite_test.go | 202 +++++++++++++++++ internal/dnsforward/dns.go | 8 +- internal/dnsforward/dnsrewrite.go | 79 +++++++ internal/dnsforward/filter.go | 16 +- internal/dnsforward/msg.go | 27 ++- internal/dnsforward/stats.go | 2 +- internal/home/controlfiltering.go | 10 +- internal/querylog/decode.go | 300 +++++++++++++++++++++++++- internal/querylog/decode_test.go | 150 ++++++++++--- internal/querylog/searchcriteria.go | 10 +- openapi/CHANGELOG.md | 42 +++- openapi/openapi.yaml | 18 +- scripts/go-lint.sh | 2 +- scripts/translations/README.md | 2 +- 23 files changed, 1010 insertions(+), 170 deletions(-) create mode 100644 internal/dnsfilter/dnsrewrite.go create mode 100644 internal/dnsfilter/dnsrewrite_test.go create mode 100644 internal/dnsforward/dnsrewrite.go diff --git a/.githooks/pre-commit b/.githooks/pre-commit index f2bbf1c9ac9..278d7d3ed9e 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -2,12 +2,12 @@ set -e -f -u -if [ "$(git diff --cached --name-only '*.js')" ] +if [ "$(git diff --cached --name-only -- '*.js')" ] then make js-lint js-test fi -if [ "$(git diff --cached --name-only '*.go')" ] +if [ "$(git diff --cached --name-only -- '*.go' 'go.mod')" ] then make go-lint go-test fi diff --git a/CHANGELOG.md b/CHANGELOG.md index 3fdf434a358..e63c913cd09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to ### Added +- `$dnsrewrite` modifier for filters ([#2102]). - The host checking API and the query logs API can now return multiple matched rules ([#2102]). - Detecting of network interface configured to have static IP address via diff --git a/Makefile b/Makefile index 1848fe34dcd..2bdee3550b0 100644 --- a/Makefile +++ b/Makefile @@ -35,6 +35,7 @@ GPG_KEY := devteam@adguard.com GPG_KEY_PASSPHRASE := GPG_CMD := gpg --detach-sig --default-key $(GPG_KEY) --pinentry-mode loopback --passphrase $(GPG_KEY_PASSPHRASE) VERBOSE := -v +REBUILD_CLIENT = 1 # See release target DIST_DIR=dist @@ -124,7 +125,8 @@ all: build init: git config core.hooksPath .githooks -build: client_with_deps +build: + test '$(REBUILD_CLIENT)' = '1' && $(MAKE) client_with_deps || exit 0 $(GO) mod download PATH=$(GOPATH)/bin:$(PATH) $(GO) generate ./... CGO_ENABLED=0 $(GO) build -ldflags="-s -w -X main.version=$(VERSION) -X main.channel=$(CHANNEL) -X main.goarm=$(GOARM)" diff --git a/README.md b/README.md index aa854f9e6c4..5af39b8ec40 100644 --- a/README.md +++ b/README.md @@ -255,7 +255,7 @@ curl -sSL https://raw.githubusercontent.com/AdguardTeam/AdGuardHome/master/scrip * Beta channel builds * Linux: [64-bit](https://static.adguard.com/adguardhome/beta/AdGuardHome_linux_amd64.tar.gz), [32-bit](https://static.adguard.com/adguardhome/beta/AdGuardHome_linux_386.tar.gz) - * Linux ARM: [32-bit ARMv6](https://static.adguard.com/adguardhome/beta/AdGuardHome_linux_armv6.tar.gz) (recommended for Rapsberry Pi), [64-bit](https://static.adguard.com/adguardhome/beta/AdGuardHome_linux_arm64.tar.gz), [32-bit ARMv5](https://static.adguard.com/adguardhome/beta/AdGuardHome_linux_armv5.tar.gz), [32-bit ARMv7](https://static.adguard.com/adguardhome/beta/AdGuardHome_linux_armv7.tar.gz) + * Linux ARM: [32-bit ARMv6](https://static.adguard.com/adguardhome/beta/AdGuardHome_linux_armv6.tar.gz) (recommended for Raspberry Pi), [64-bit](https://static.adguard.com/adguardhome/beta/AdGuardHome_linux_arm64.tar.gz), [32-bit ARMv5](https://static.adguard.com/adguardhome/beta/AdGuardHome_linux_armv5.tar.gz), [32-bit ARMv7](https://static.adguard.com/adguardhome/beta/AdGuardHome_linux_armv7.tar.gz) * Linux MIPS: [32-bit MIPS](https://static.adguard.com/adguardhome/beta/AdGuardHome_linux_mips_softfloat.tar.gz), [32-bit MIPSLE](https://static.adguard.com/adguardhome/beta/AdGuardHome_linux_mipsle_softfloat.tar.gz), [64-bit MIPS](https://static.adguard.com/adguardhome/beta/AdGuardHome_linux_mips64_softfloat.tar.gz), [64-bit MIPSLE](https://static.adguard.com/adguardhome/beta/AdGuardHome_linux_mips64le_softfloat.tar.gz) * Windows: [64-bit](https://static.adguard.com/adguardhome/beta/AdGuardHome_windows_amd64.zip), [32-bit](https://static.adguard.com/adguardhome/beta/AdGuardHome_windows_386.zip) * MacOS: [64-bit](https://static.adguard.com/adguardhome/beta/AdGuardHome_darwin_amd64.zip), [32-bit](https://static.adguard.com/adguardhome/beta/AdGuardHome_darwin_386.zip) @@ -264,7 +264,7 @@ curl -sSL https://raw.githubusercontent.com/AdguardTeam/AdGuardHome/master/scrip * Edge channel builds * Linux: [64-bit](https://static.adguard.com/adguardhome/edge/AdGuardHome_linux_amd64.tar.gz), [32-bit](https://static.adguard.com/adguardhome/edge/AdGuardHome_linux_386.tar.gz) - * Linux ARM: [32-bit ARMv6](https://static.adguard.com/adguardhome/edge/AdGuardHome_linux_armv6.tar.gz) (recommended for Rapsberry Pi), [64-bit](https://static.adguard.com/adguardhome/edge/AdGuardHome_linux_arm64.tar.gz), [32-bit ARMv5](https://static.adguard.com/adguardhome/edge/AdGuardHome_linux_armv5.tar.gz), [32-bit ARMv7](https://static.adguard.com/adguardhome/edge/AdGuardHome_linux_armv7.tar.gz) + * Linux ARM: [32-bit ARMv6](https://static.adguard.com/adguardhome/edge/AdGuardHome_linux_armv6.tar.gz) (recommended for Raspberry Pi), [64-bit](https://static.adguard.com/adguardhome/edge/AdGuardHome_linux_arm64.tar.gz), [32-bit ARMv5](https://static.adguard.com/adguardhome/edge/AdGuardHome_linux_armv5.tar.gz), [32-bit ARMv7](https://static.adguard.com/adguardhome/edge/AdGuardHome_linux_armv7.tar.gz) * Linux MIPS: [32-bit MIPS](https://static.adguard.com/adguardhome/edge/AdGuardHome_linux_mips_softfloat.tar.gz), [32-bit MIPSLE](https://static.adguard.com/adguardhome/edge/AdGuardHome_linux_mipsle_softfloat.tar.gz), [64-bit MIPS](https://static.adguard.com/adguardhome/edge/AdGuardHome_linux_mips64_softfloat.tar.gz), [64-bit MIPSLE](https://static.adguard.com/adguardhome/edge/AdGuardHome_linux_mips64le_softfloat.tar.gz) * Windows: [64-bit](https://static.adguard.com/adguardhome/edge/AdGuardHome_windows_amd64.zip), [32-bit](https://static.adguard.com/adguardhome/edge/AdGuardHome_windows_386.zip) * MacOS: [64-bit](https://static.adguard.com/adguardhome/edge/AdGuardHome_darwin_amd64.zip), [32-bit](https://static.adguard.com/adguardhome/edge/AdGuardHome_darwin_386.zip) diff --git a/go.mod b/go.mod index 81d2e24864d..10866d88eba 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.14 require ( github.com/AdguardTeam/dnsproxy v0.33.7 github.com/AdguardTeam/golibs v0.4.4 - github.com/AdguardTeam/urlfilter v0.13.0 + github.com/AdguardTeam/urlfilter v0.14.0 github.com/NYTimes/gziphandler v1.1.1 github.com/ameshkov/dnscrypt/v2 v2.0.1 github.com/fsnotify/fsnotify v1.4.9 diff --git a/go.sum b/go.sum index 37dd5867416..764a2fa00d8 100644 --- a/go.sum +++ b/go.sum @@ -26,8 +26,8 @@ github.com/AdguardTeam/golibs v0.4.2/go.mod h1:skKsDKIBB7kkFflLJBpfGX+G8QFTx0WKU github.com/AdguardTeam/golibs v0.4.4 h1:cM9UySQiYFW79zo5XRwnaIWVzfW4eNXmZktMrWbthpw= github.com/AdguardTeam/golibs v0.4.4/go.mod h1:skKsDKIBB7kkFflLJBpfGX+G8QFTx0WKUzB6TIgtUj4= github.com/AdguardTeam/gomitmproxy v0.2.0/go.mod h1:Qdv0Mktnzer5zpdpi5rAwixNJzW2FN91LjKJCkVbYGU= -github.com/AdguardTeam/urlfilter v0.13.0 h1:MfO46K81JVTkhgP6gRu/buKl5wAOSfusjiDwjT1JN1c= -github.com/AdguardTeam/urlfilter v0.13.0/go.mod h1:klx4JbOfc4EaNb5lWLqOwfg+pVcyRukmoJRvO55lL5U= +github.com/AdguardTeam/urlfilter v0.14.0 h1:+aAhOvZDVGzl5gTERB4pOJCL1zxMyw7vLecJJ6TQTCw= +github.com/AdguardTeam/urlfilter v0.14.0/go.mod h1:klx4JbOfc4EaNb5lWLqOwfg+pVcyRukmoJRvO55lL5U= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= diff --git a/internal/dnsfilter/dnsfilter.go b/internal/dnsfilter/dnsfilter.go index 1735154d8ec..4a1b255e5bc 100644 --- a/internal/dnsfilter/dnsfilter.go +++ b/internal/dnsfilter/dnsfilter.go @@ -1,4 +1,4 @@ -// Package dnsfilter implements a DNS filter. +// Package dnsfilter implements a DNS request and response filter. package dnsfilter import ( @@ -95,8 +95,8 @@ type filtersInitializerParams struct { type DNSFilter struct { rulesStorage *filterlist.RuleStorage filteringEngine *urlfilter.DNSEngine - rulesStorageWhite *filterlist.RuleStorage - filteringEngineWhite *urlfilter.DNSEngine + rulesStorageAllow *filterlist.RuleStorage + filteringEngineAllow *urlfilter.DNSEngine engineLock sync.RWMutex parentalServer string // access via methods @@ -127,16 +127,16 @@ const ( // NotFilteredNotFound - host was not find in any checks, default value for result NotFilteredNotFound Reason = iota - // NotFilteredWhiteList - the host is explicitly whitelisted - NotFilteredWhiteList + // NotFilteredAllowList - the host is explicitly allowed + NotFilteredAllowList // NotFilteredError is returned when there was an error during // checking. Reserved, currently unused. NotFilteredError // reasons for filtering - // FilteredBlackList - the host was matched to be advertising host - FilteredBlackList + // FilteredBlockList - the host was matched to be advertising host + FilteredBlockList // FilteredSafeBrowsing - the host was matched to be malicious/phishing FilteredSafeBrowsing // FilteredParental - the host was matched to be outside of parental control settings @@ -155,16 +155,20 @@ const ( // RewriteAutoHosts is returned when there was a rewrite by // autohosts rules (/etc/hosts and so on). RewriteAutoHosts + + // DNSRewriteRule is returned when a $dnsrewrite filter rule was + // applied. + DNSRewriteRule ) // TODO(a.garipov): Resync with actual code names or replace completely // in HTTP API v1. var reasonNames = []string{ NotFilteredNotFound: "NotFilteredNotFound", - NotFilteredWhiteList: "NotFilteredWhiteList", + NotFilteredAllowList: "NotFilteredWhiteList", NotFilteredError: "NotFilteredError", - FilteredBlackList: "FilteredBlackList", + FilteredBlockList: "FilteredBlackList", FilteredSafeBrowsing: "FilteredSafeBrowsing", FilteredParental: "FilteredParental", FilteredInvalid: "FilteredInvalid", @@ -174,12 +178,15 @@ var reasonNames = []string{ ReasonRewrite: "Rewrite", RewriteAutoHosts: "RewriteEtcHosts", + + DNSRewriteRule: "DNSRewriteRule", } func (r Reason) String() string { - if uint(r) >= uint(len(reasonNames)) { + if r < 0 || int(r) >= len(reasonNames) { return "" } + return reasonNames[r] } @@ -278,16 +285,15 @@ func (d *DNSFilter) reset() { } } - if d.rulesStorageWhite != nil { - err = d.rulesStorageWhite.Close() + if d.rulesStorageAllow != nil { + err = d.rulesStorageAllow.Close() if err != nil { - log.Error("dnsfilter: rulesStorageWhite.Close: %s", err) + log.Error("dnsfilter: rulesStorageAllow.Close: %s", err) } } } type dnsFilterContext struct { - stats Stats safebrowsingCache cache.Cache parentalCache cache.Cache safeSearchCache cache.Cache @@ -339,6 +345,9 @@ type Result struct { // ServiceName is the name of the blocked service. It is empty // unless Reason is set to FilteredBlockedService. ServiceName string `json:",omitempty"` + + // DNSRewriteResult is the $dnsrewrite filter rule result. + DNSRewriteResult *DNSRewriteResult `json:",omitempty"` } // Matched returns true if any match at all was found regardless of @@ -383,9 +392,6 @@ func (d *DNSFilter) CheckHost(host string, qtype uint16, setts *RequestFiltering } } - // Then check the filter lists. - // if request is blocked -- it should be blocked. - // if it is whitelisted -- we should do nothing with it anymore. if setts.FilteringEnabled { result, err = d.matchHost(host, qtype, *setts) if err != nil { @@ -476,9 +482,7 @@ func (d *DNSFilter) checkAutoHosts(host string, qtype uint16, result *Result) (m // . repeat for the new domain name (Note: we return only the last CNAME) // . Find A or AAAA record for a domain name (exact match or by wildcard) // . if found, set IP addresses (IPv4 or IPv6 depending on qtype) in Result.IPList array -func (d *DNSFilter) processRewrites(host string, qtype uint16) Result { - var res Result - +func (d *DNSFilter) processRewrites(host string, qtype uint16) (res Result) { d.confLock.RLock() defer d.confLock.RUnlock() @@ -493,7 +497,8 @@ func (d *DNSFilter) processRewrites(host string, qtype uint16) Result { log.Debug("Rewrite: CNAME for %s is %s", host, rr[0].Answer) if host == rr[0].Answer { // "host == CNAME" is an exception - res.Reason = 0 + res.Reason = NotFilteredNotFound + return res } @@ -616,7 +621,7 @@ func (d *DNSFilter) initFiltering(allowFilters, blockFilters []Filter) error { if err != nil { return err } - rulesStorageWhite, filteringEngineWhite, err := createFilteringEngine(allowFilters) + rulesStorageAllow, filteringEngineAllow, err := createFilteringEngine(allowFilters) if err != nil { return err } @@ -625,8 +630,8 @@ func (d *DNSFilter) initFiltering(allowFilters, blockFilters []Filter) error { d.reset() d.rulesStorage = rulesStorage d.filteringEngine = filteringEngine - d.rulesStorageWhite = rulesStorageWhite - d.filteringEngineWhite = filteringEngineWhite + d.rulesStorageAllow = rulesStorageAllow + d.filteringEngineAllow = filteringEngineAllow d.engineLock.Unlock() // Make sure that the OS reclaims memory as soon as possible @@ -636,9 +641,31 @@ func (d *DNSFilter) initFiltering(allowFilters, blockFilters []Filter) error { return nil } +// matchHostProcessAllowList processes the allowlist logic of host +// matching. +func (d *DNSFilter) matchHostProcessAllowList(host string, dnsres urlfilter.DNSResult) (res Result, err error) { + var rule rules.Rule + if dnsres.NetworkRule != nil { + rule = dnsres.NetworkRule + } else if len(dnsres.HostRulesV4) > 0 { + rule = dnsres.HostRulesV4[0] + } else if len(dnsres.HostRulesV6) > 0 { + rule = dnsres.HostRulesV6[0] + } + + if rule == nil { + return Result{}, fmt.Errorf("invalid dns result: rules are empty") + } + + log.Debug("Filtering: found allowlist rule for host %q: %q list_id: %d", + host, rule.Text(), rule.GetFilterListID()) + + return makeResult(rule, NotFilteredAllowList), nil +} + // matchHost is a low-level way to check only if hostname is filtered by rules, // skipping expensive safebrowsing and parental lookups. -func (d *DNSFilter) matchHost(host string, qtype uint16, setts RequestFilteringSettings) (Result, error) { +func (d *DNSFilter) matchHost(host string, qtype uint16, setts RequestFilteringSettings) (res Result, err error) { d.engineLock.RLock() // Keep in mind that this lock must be held no just when calling Match() // but also while using the rules returned by it. @@ -652,22 +679,10 @@ func (d *DNSFilter) matchHost(host string, qtype uint16, setts RequestFilteringS DNSType: qtype, } - if d.filteringEngineWhite != nil { - rr, ok := d.filteringEngineWhite.MatchRequest(ureq) + if d.filteringEngineAllow != nil { + dnsres, ok := d.filteringEngineAllow.MatchRequest(ureq) if ok { - var rule rules.Rule - if rr.NetworkRule != nil { - rule = rr.NetworkRule - } else if rr.HostRulesV4 != nil { - rule = rr.HostRulesV4[0] - } else if rr.HostRulesV6 != nil { - rule = rr.HostRulesV6[0] - } - - log.Debug("Filtering: found whitelist rule for host %q: %q list_id: %d", - host, rule.Text(), rule.GetFilterListID()) - res := makeResult(rule, NotFilteredWhiteList) - return res, nil + return d.matchHostProcessAllowList(host, dnsres) } } @@ -675,54 +690,65 @@ func (d *DNSFilter) matchHost(host string, qtype uint16, setts RequestFilteringS return Result{}, nil } - rr, ok := d.filteringEngine.MatchRequest(ureq) - if !ok { + dnsres, ok := d.filteringEngine.MatchRequest(ureq) + + // Check DNS rewrites first, because the API there is a bit + // awkward. + if dnsr := dnsres.DNSRewrites(); len(dnsr) > 0 { + res = d.processDNSRewrites(dnsr) + if res.Reason == DNSRewriteRule && res.CanonName == host { + // A rewrite of a host to itself. Go on and + // try matching other things. + } else { + return res, nil + } + } else if !ok { return Result{}, nil } - if rr.NetworkRule != nil { + if dnsres.NetworkRule != nil { log.Debug("Filtering: found rule for host %q: %q list_id: %d", - host, rr.NetworkRule.Text(), rr.NetworkRule.GetFilterListID()) - reason := FilteredBlackList - if rr.NetworkRule.Whitelist { - reason = NotFilteredWhiteList + host, dnsres.NetworkRule.Text(), dnsres.NetworkRule.GetFilterListID()) + reason := FilteredBlockList + if dnsres.NetworkRule.Whitelist { + reason = NotFilteredAllowList } - res := makeResult(rr.NetworkRule, reason) - return res, nil + + return makeResult(dnsres.NetworkRule, reason), nil } - if qtype == dns.TypeA && rr.HostRulesV4 != nil { - rule := rr.HostRulesV4[0] // note that we process only 1 matched rule + if qtype == dns.TypeA && dnsres.HostRulesV4 != nil { + rule := dnsres.HostRulesV4[0] // note that we process only 1 matched rule log.Debug("Filtering: found rule for host %q: %q list_id: %d", host, rule.Text(), rule.GetFilterListID()) - res := makeResult(rule, FilteredBlackList) + res = makeResult(rule, FilteredBlockList) res.Rules[0].IP = rule.IP.To4() return res, nil } - if qtype == dns.TypeAAAA && rr.HostRulesV6 != nil { - rule := rr.HostRulesV6[0] // note that we process only 1 matched rule + if qtype == dns.TypeAAAA && dnsres.HostRulesV6 != nil { + rule := dnsres.HostRulesV6[0] // note that we process only 1 matched rule log.Debug("Filtering: found rule for host %q: %q list_id: %d", host, rule.Text(), rule.GetFilterListID()) - res := makeResult(rule, FilteredBlackList) + res = makeResult(rule, FilteredBlockList) res.Rules[0].IP = rule.IP return res, nil } - if rr.HostRulesV4 != nil || rr.HostRulesV6 != nil { + if dnsres.HostRulesV4 != nil || dnsres.HostRulesV6 != nil { // Question Type doesn't match the host rules // Return the first matched host rule, but without an IP address var rule rules.Rule - if rr.HostRulesV4 != nil { - rule = rr.HostRulesV4[0] - } else if rr.HostRulesV6 != nil { - rule = rr.HostRulesV6[0] + if dnsres.HostRulesV4 != nil { + rule = dnsres.HostRulesV4[0] + } else if dnsres.HostRulesV6 != nil { + rule = dnsres.HostRulesV6[0] } log.Debug("Filtering: found rule for host %q: %q list_id: %d", host, rule.Text(), rule.GetFilterListID()) - res := makeResult(rule, FilteredBlackList) + res = makeResult(rule, FilteredBlockList) res.Rules[0].IP = net.IP{} return res, nil @@ -741,7 +767,7 @@ func makeResult(rule rules.Rule, reason Reason) Result { }}, } - if reason == FilteredBlackList { + if reason == FilteredBlockList { res.IsFiltered = true } diff --git a/internal/dnsfilter/dnsfilter_test.go b/internal/dnsfilter/dnsfilter_test.go index 96376162b44..2bae12de07e 100644 --- a/internal/dnsfilter/dnsfilter_test.go +++ b/internal/dnsfilter/dnsfilter_test.go @@ -178,7 +178,6 @@ func TestSafeBrowsing(t *testing.T) { d := NewForTest(&Config{SafeBrowsingEnabled: true}, nil) defer d.Close() - gctx.stats.Safebrowsing.Requests = 0 d.checkMatch(t, "wmconvirus.narod.ru") assert.True(t, strings.Contains(logOutput.String(), "SafeBrowsing lookup for wmconvirus.narod.ru")) @@ -366,7 +365,7 @@ const nl = "\n" const ( blockingRules = `||example.org^` + nl - whitelistRules = `||example.org^` + nl + `@@||test.example.org` + nl + allowlistRules = `||example.org^` + nl + `@@||test.example.org` + nl importantRules = `@@||example.org^` + nl + `||test.example.org^$important` + nl regexRules = `/example\.org/` + nl + `@@||test.example.org^` + nl maskRules = `test*.example.org^` + nl + `exam*.com` + nl @@ -381,49 +380,49 @@ var tests = []struct { reason Reason dnsType uint16 }{ - {"sanity", "||doubleclick.net^", "www.doubleclick.net", true, FilteredBlackList, dns.TypeA}, + {"sanity", "||doubleclick.net^", "www.doubleclick.net", true, FilteredBlockList, dns.TypeA}, {"sanity", "||doubleclick.net^", "nodoubleclick.net", false, NotFilteredNotFound, dns.TypeA}, {"sanity", "||doubleclick.net^", "doubleclick.net.ru", false, NotFilteredNotFound, dns.TypeA}, {"sanity", "||doubleclick.net^", "wmconvirus.narod.ru", false, NotFilteredNotFound, dns.TypeA}, - {"blocking", blockingRules, "example.org", true, FilteredBlackList, dns.TypeA}, - {"blocking", blockingRules, "test.example.org", true, FilteredBlackList, dns.TypeA}, - {"blocking", blockingRules, "test.test.example.org", true, FilteredBlackList, dns.TypeA}, + {"blocking", blockingRules, "example.org", true, FilteredBlockList, dns.TypeA}, + {"blocking", blockingRules, "test.example.org", true, FilteredBlockList, dns.TypeA}, + {"blocking", blockingRules, "test.test.example.org", true, FilteredBlockList, dns.TypeA}, {"blocking", blockingRules, "testexample.org", false, NotFilteredNotFound, dns.TypeA}, {"blocking", blockingRules, "onemoreexample.org", false, NotFilteredNotFound, dns.TypeA}, - {"whitelist", whitelistRules, "example.org", true, FilteredBlackList, dns.TypeA}, - {"whitelist", whitelistRules, "test.example.org", false, NotFilteredWhiteList, dns.TypeA}, - {"whitelist", whitelistRules, "test.test.example.org", false, NotFilteredWhiteList, dns.TypeA}, - {"whitelist", whitelistRules, "testexample.org", false, NotFilteredNotFound, dns.TypeA}, - {"whitelist", whitelistRules, "onemoreexample.org", false, NotFilteredNotFound, dns.TypeA}, + {"allowlist", allowlistRules, "example.org", true, FilteredBlockList, dns.TypeA}, + {"allowlist", allowlistRules, "test.example.org", false, NotFilteredAllowList, dns.TypeA}, + {"allowlist", allowlistRules, "test.test.example.org", false, NotFilteredAllowList, dns.TypeA}, + {"allowlist", allowlistRules, "testexample.org", false, NotFilteredNotFound, dns.TypeA}, + {"allowlist", allowlistRules, "onemoreexample.org", false, NotFilteredNotFound, dns.TypeA}, - {"important", importantRules, "example.org", false, NotFilteredWhiteList, dns.TypeA}, - {"important", importantRules, "test.example.org", true, FilteredBlackList, dns.TypeA}, - {"important", importantRules, "test.test.example.org", true, FilteredBlackList, dns.TypeA}, + {"important", importantRules, "example.org", false, NotFilteredAllowList, dns.TypeA}, + {"important", importantRules, "test.example.org", true, FilteredBlockList, dns.TypeA}, + {"important", importantRules, "test.test.example.org", true, FilteredBlockList, dns.TypeA}, {"important", importantRules, "testexample.org", false, NotFilteredNotFound, dns.TypeA}, {"important", importantRules, "onemoreexample.org", false, NotFilteredNotFound, dns.TypeA}, - {"regex", regexRules, "example.org", true, FilteredBlackList, dns.TypeA}, - {"regex", regexRules, "test.example.org", false, NotFilteredWhiteList, dns.TypeA}, - {"regex", regexRules, "test.test.example.org", false, NotFilteredWhiteList, dns.TypeA}, - {"regex", regexRules, "testexample.org", true, FilteredBlackList, dns.TypeA}, - {"regex", regexRules, "onemoreexample.org", true, FilteredBlackList, dns.TypeA}, - - {"mask", maskRules, "test.example.org", true, FilteredBlackList, dns.TypeA}, - {"mask", maskRules, "test2.example.org", true, FilteredBlackList, dns.TypeA}, - {"mask", maskRules, "example.com", true, FilteredBlackList, dns.TypeA}, - {"mask", maskRules, "exampleeee.com", true, FilteredBlackList, dns.TypeA}, - {"mask", maskRules, "onemoreexamsite.com", true, FilteredBlackList, dns.TypeA}, + {"regex", regexRules, "example.org", true, FilteredBlockList, dns.TypeA}, + {"regex", regexRules, "test.example.org", false, NotFilteredAllowList, dns.TypeA}, + {"regex", regexRules, "test.test.example.org", false, NotFilteredAllowList, dns.TypeA}, + {"regex", regexRules, "testexample.org", true, FilteredBlockList, dns.TypeA}, + {"regex", regexRules, "onemoreexample.org", true, FilteredBlockList, dns.TypeA}, + + {"mask", maskRules, "test.example.org", true, FilteredBlockList, dns.TypeA}, + {"mask", maskRules, "test2.example.org", true, FilteredBlockList, dns.TypeA}, + {"mask", maskRules, "example.com", true, FilteredBlockList, dns.TypeA}, + {"mask", maskRules, "exampleeee.com", true, FilteredBlockList, dns.TypeA}, + {"mask", maskRules, "onemoreexamsite.com", true, FilteredBlockList, dns.TypeA}, {"mask", maskRules, "example.org", false, NotFilteredNotFound, dns.TypeA}, {"mask", maskRules, "testexample.org", false, NotFilteredNotFound, dns.TypeA}, {"mask", maskRules, "example.co.uk", false, NotFilteredNotFound, dns.TypeA}, {"dnstype", dnstypeRules, "onemoreexample.org", false, NotFilteredNotFound, dns.TypeA}, {"dnstype", dnstypeRules, "example.org", false, NotFilteredNotFound, dns.TypeA}, - {"dnstype", dnstypeRules, "example.org", true, FilteredBlackList, dns.TypeAAAA}, - {"dnstype", dnstypeRules, "test.example.org", false, NotFilteredWhiteList, dns.TypeA}, - {"dnstype", dnstypeRules, "test.example.org", false, NotFilteredWhiteList, dns.TypeAAAA}, + {"dnstype", dnstypeRules, "example.org", true, FilteredBlockList, dns.TypeAAAA}, + {"dnstype", dnstypeRules, "test.example.org", false, NotFilteredAllowList, dns.TypeA}, + {"dnstype", dnstypeRules, "test.example.org", false, NotFilteredAllowList, dns.TypeAAAA}, } func TestMatching(t *testing.T) { @@ -470,7 +469,7 @@ func TestWhitelist(t *testing.T) { // matched by white filter res, err := d.CheckHost("host1", dns.TypeA, &setts) assert.True(t, err == nil) - assert.True(t, !res.IsFiltered && res.Reason == NotFilteredWhiteList) + assert.True(t, !res.IsFiltered && res.Reason == NotFilteredAllowList) if assert.Len(t, res.Rules, 1) { assert.True(t, res.Rules[0].Text == "||host1^") } @@ -478,7 +477,7 @@ func TestWhitelist(t *testing.T) { // not matched by white filter, but matched by block filter res, err = d.CheckHost("host2", dns.TypeA, &setts) assert.True(t, err == nil) - assert.True(t, res.IsFiltered && res.Reason == FilteredBlackList) + assert.True(t, res.IsFiltered && res.Reason == FilteredBlockList) if assert.Len(t, res.Rules, 1) { assert.True(t, res.Rules[0].Text == "||host2^") } @@ -512,8 +511,8 @@ func TestClientSettings(t *testing.T) { // blocked by filters r, _ = d.CheckHost("example.org", dns.TypeA, &setts) - if !r.IsFiltered || r.Reason != FilteredBlackList { - t.Fatalf("CheckHost FilteredBlackList") + if !r.IsFiltered || r.Reason != FilteredBlockList { + t.Fatalf("CheckHost FilteredBlockList") } // blocked by parental diff --git a/internal/dnsfilter/dnsrewrite.go b/internal/dnsfilter/dnsrewrite.go new file mode 100644 index 00000000000..1239fbad5e7 --- /dev/null +++ b/internal/dnsfilter/dnsrewrite.go @@ -0,0 +1,80 @@ +package dnsfilter + +import ( + "github.com/AdguardTeam/urlfilter/rules" + "github.com/miekg/dns" +) + +// DNSRewriteResult is the result of application of $dnsrewrite rules. +type DNSRewriteResult struct { + RCode rules.RCode `json:",omitempty"` + Response DNSRewriteResultResponse `json:",omitempty"` +} + +// DNSRewriteResultResponse is the collection of DNS response records +// the server returns. +type DNSRewriteResultResponse map[rules.RRType][]rules.RRValue + +// processDNSRewrites processes DNS rewrite rules in dnsr. It returns +// an empty result if dnsr is empty. Otherwise, the result will have +// either CanonName or DNSRewriteResult set. +func (d *DNSFilter) processDNSRewrites(dnsr []*rules.NetworkRule) (res Result) { + if len(dnsr) == 0 { + return Result{} + } + + var rules []*ResultRule + dnsrr := &DNSRewriteResult{ + Response: DNSRewriteResultResponse{}, + } + + for _, nr := range dnsr { + dr := nr.DNSRewrite + if dr.NewCNAME != "" { + // NewCNAME rules have a higher priority than + // the other rules. + rules := []*ResultRule{{ + FilterListID: int64(nr.GetFilterListID()), + Text: nr.RuleText, + }} + + return Result{ + Reason: DNSRewriteRule, + Rules: rules, + CanonName: dr.NewCNAME, + } + } + + switch dr.RCode { + case dns.RcodeSuccess: + dnsrr.RCode = dr.RCode + dnsrr.Response[dr.RRType] = append(dnsrr.Response[dr.RRType], dr.Value) + rules = append(rules, &ResultRule{ + FilterListID: int64(nr.GetFilterListID()), + Text: nr.RuleText, + }) + default: + // RcodeRefused and other such codes have higher + // priority. Return immediately. + rules := []*ResultRule{{ + FilterListID: int64(nr.GetFilterListID()), + Text: nr.RuleText, + }} + dnsrr = &DNSRewriteResult{ + RCode: dr.RCode, + } + + return Result{ + Reason: DNSRewriteRule, + Rules: rules, + DNSRewriteResult: dnsrr, + } + } + } + + return Result{ + Reason: DNSRewriteRule, + Rules: rules, + DNSRewriteResult: dnsrr, + } +} diff --git a/internal/dnsfilter/dnsrewrite_test.go b/internal/dnsfilter/dnsrewrite_test.go new file mode 100644 index 00000000000..4918ccc05d4 --- /dev/null +++ b/internal/dnsfilter/dnsrewrite_test.go @@ -0,0 +1,202 @@ +package dnsfilter + +import ( + "net" + "path" + "testing" + + "github.com/miekg/dns" + "github.com/stretchr/testify/assert" +) + +func TestDNSFilter_CheckHostRules_dnsrewrite(t *testing.T) { + const text = ` +|cname^$dnsrewrite=new_cname + +|a_record^$dnsrewrite=127.0.0.1 + +|aaaa_record^$dnsrewrite=::1 + +|txt_record^$dnsrewrite=NOERROR;TXT;hello_world + +|refused^$dnsrewrite=REFUSED + +|a_records^$dnsrewrite=127.0.0.1 +|a_records^$dnsrewrite=127.0.0.2 + +|aaaa_records^$dnsrewrite=::1 +|aaaa_records^$dnsrewrite=::2 + +|disable_one^$dnsrewrite=127.0.0.1 +|disable_one^$dnsrewrite=127.0.0.2 +@@||disable_one^$dnsrewrite=127.0.0.1 + +|disable_cname^$dnsrewrite=127.0.0.1 +|disable_cname^$dnsrewrite=new_cname +@@||disable_cname^$dnsrewrite=new_cname + +|disable_cname_many^$dnsrewrite=127.0.0.1 +|disable_cname_many^$dnsrewrite=new_cname_1 +|disable_cname_many^$dnsrewrite=new_cname_2 +@@||disable_cname_many^$dnsrewrite=new_cname_1 + +|disable_all^$dnsrewrite=127.0.0.1 +|disable_all^$dnsrewrite=127.0.0.2 +@@||disable_all^$dnsrewrite +` + f := NewForTest(nil, []Filter{{ID: 0, Data: []byte(text)}}) + setts := &RequestFilteringSettings{ + FilteringEnabled: true, + } + + ipv4p1 := net.IPv4(127, 0, 0, 1) + ipv4p2 := net.IPv4(127, 0, 0, 2) + ipv6p1 := net.IP{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1} + ipv6p2 := net.IP{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2} + + t.Run("cname", func(t *testing.T) { + dtyp := dns.TypeA + host := path.Base(t.Name()) + + res, err := f.CheckHostRules(host, dtyp, setts) + assert.Nil(t, err) + assert.Equal(t, "new_cname", res.CanonName) + }) + + t.Run("a_record", func(t *testing.T) { + dtyp := dns.TypeA + host := path.Base(t.Name()) + + res, err := f.CheckHostRules(host, dtyp, setts) + assert.Nil(t, err) + + if dnsrr := res.DNSRewriteResult; assert.NotNil(t, dnsrr) { + assert.Equal(t, dns.RcodeSuccess, dnsrr.RCode) + if ipVals := dnsrr.Response[dtyp]; assert.Len(t, ipVals, 1) { + assert.Equal(t, ipv4p1, ipVals[0]) + } + } + }) + + t.Run("aaaa_record", func(t *testing.T) { + dtyp := dns.TypeAAAA + host := path.Base(t.Name()) + + res, err := f.CheckHostRules(host, dtyp, setts) + assert.Nil(t, err) + + if dnsrr := res.DNSRewriteResult; assert.NotNil(t, dnsrr) { + assert.Equal(t, dns.RcodeSuccess, dnsrr.RCode) + if ipVals := dnsrr.Response[dtyp]; assert.Len(t, ipVals, 1) { + assert.Equal(t, ipv6p1, ipVals[0]) + } + } + }) + + t.Run("txt_record", func(t *testing.T) { + dtyp := dns.TypeTXT + host := path.Base(t.Name()) + res, err := f.CheckHostRules(host, dtyp, setts) + assert.Nil(t, err) + + if dnsrr := res.DNSRewriteResult; assert.NotNil(t, dnsrr) { + assert.Equal(t, dns.RcodeSuccess, dnsrr.RCode) + if strVals := dnsrr.Response[dtyp]; assert.Len(t, strVals, 1) { + assert.Equal(t, "hello_world", strVals[0]) + } + } + }) + + t.Run("refused", func(t *testing.T) { + host := path.Base(t.Name()) + res, err := f.CheckHostRules(host, dns.TypeA, setts) + assert.Nil(t, err) + + if dnsrr := res.DNSRewriteResult; assert.NotNil(t, dnsrr) { + assert.Equal(t, dns.RcodeRefused, dnsrr.RCode) + } + }) + + t.Run("a_records", func(t *testing.T) { + dtyp := dns.TypeA + host := path.Base(t.Name()) + + res, err := f.CheckHostRules(host, dtyp, setts) + assert.Nil(t, err) + + if dnsrr := res.DNSRewriteResult; assert.NotNil(t, dnsrr) { + assert.Equal(t, dns.RcodeSuccess, dnsrr.RCode) + if ipVals := dnsrr.Response[dtyp]; assert.Len(t, ipVals, 2) { + assert.Equal(t, ipv4p1, ipVals[0]) + assert.Equal(t, ipv4p2, ipVals[1]) + } + } + }) + + t.Run("aaaa_records", func(t *testing.T) { + dtyp := dns.TypeAAAA + host := path.Base(t.Name()) + + res, err := f.CheckHostRules(host, dtyp, setts) + assert.Nil(t, err) + + if dnsrr := res.DNSRewriteResult; assert.NotNil(t, dnsrr) { + assert.Equal(t, dns.RcodeSuccess, dnsrr.RCode) + if ipVals := dnsrr.Response[dtyp]; assert.Len(t, ipVals, 2) { + assert.Equal(t, ipv6p1, ipVals[0]) + assert.Equal(t, ipv6p2, ipVals[1]) + } + } + }) + + t.Run("disable_one", func(t *testing.T) { + dtyp := dns.TypeA + host := path.Base(t.Name()) + + res, err := f.CheckHostRules(host, dtyp, setts) + assert.Nil(t, err) + + if dnsrr := res.DNSRewriteResult; assert.NotNil(t, dnsrr) { + assert.Equal(t, dns.RcodeSuccess, dnsrr.RCode) + if ipVals := dnsrr.Response[dtyp]; assert.Len(t, ipVals, 1) { + assert.Equal(t, ipv4p2, ipVals[0]) + } + } + }) + + t.Run("disable_cname", func(t *testing.T) { + dtyp := dns.TypeA + host := path.Base(t.Name()) + + res, err := f.CheckHostRules(host, dtyp, setts) + assert.Nil(t, err) + assert.Equal(t, "", res.CanonName) + + if dnsrr := res.DNSRewriteResult; assert.NotNil(t, dnsrr) { + assert.Equal(t, dns.RcodeSuccess, dnsrr.RCode) + if ipVals := dnsrr.Response[dtyp]; assert.Len(t, ipVals, 1) { + assert.Equal(t, ipv4p1, ipVals[0]) + } + } + }) + + t.Run("disable_cname_many", func(t *testing.T) { + dtyp := dns.TypeA + host := path.Base(t.Name()) + + res, err := f.CheckHostRules(host, dtyp, setts) + assert.Nil(t, err) + assert.Equal(t, "new_cname_2", res.CanonName) + assert.Nil(t, res.DNSRewriteResult) + }) + + t.Run("disable_all", func(t *testing.T) { + dtyp := dns.TypeA + host := path.Base(t.Name()) + + res, err := f.CheckHostRules(host, dtyp, setts) + assert.Nil(t, err) + assert.Equal(t, "", res.CanonName) + assert.Len(t, res.Rules, 0) + }) +} diff --git a/internal/dnsforward/dns.go b/internal/dnsforward/dns.go index 0f9c764b7c8..d7208d1ce0d 100644 --- a/internal/dnsforward/dns.go +++ b/internal/dnsforward/dns.go @@ -366,7 +366,9 @@ func processFilteringAfterResponse(ctx *dnsContext) int { var err error switch res.Reason { - case dnsfilter.ReasonRewrite: + case dnsfilter.ReasonRewrite, + dnsfilter.DNSRewriteRule: + if len(ctx.origQuestion.Name) == 0 { // origQuestion is set in case we get only CNAME without IP from rewrites table break @@ -378,11 +380,11 @@ func processFilteringAfterResponse(ctx *dnsContext) int { if len(d.Res.Answer) != 0 { answer := []dns.RR{} answer = append(answer, s.genCNAMEAnswer(d.Req, res.CanonName)) - answer = append(answer, d.Res.Answer...) // host -> IP + answer = append(answer, d.Res.Answer...) d.Res.Answer = answer } - case dnsfilter.NotFilteredWhiteList: + case dnsfilter.NotFilteredAllowList: // nothing default: diff --git a/internal/dnsforward/dnsrewrite.go b/internal/dnsforward/dnsrewrite.go new file mode 100644 index 00000000000..0189532364e --- /dev/null +++ b/internal/dnsforward/dnsrewrite.go @@ -0,0 +1,79 @@ +package dnsforward + +import ( + "fmt" + "net" + + "github.com/AdguardTeam/AdGuardHome/internal/agherr" + "github.com/AdguardTeam/AdGuardHome/internal/dnsfilter" + "github.com/AdguardTeam/dnsproxy/proxy" + "github.com/AdguardTeam/golibs/log" + "github.com/AdguardTeam/urlfilter/rules" + "github.com/miekg/dns" +) + +// filterDNSRewriteResponse handles a single DNS rewrite response entry. +// It returns the constructed answer resource record. +func (s *Server) filterDNSRewriteResponse(req *dns.Msg, rr rules.RRType, v rules.RRValue) (ans dns.RR, err error) { + switch rr { + case dns.TypeA, dns.TypeAAAA: + ip, ok := v.(net.IP) + if !ok { + return nil, fmt.Errorf("value has type %T, not net.IP", v) + } + + if rr == dns.TypeA { + return s.genAAnswer(req, ip.To4()), nil + } + + return s.genAAAAAnswer(req, ip), nil + case dns.TypeTXT: + str, ok := v.(string) + if !ok { + return nil, fmt.Errorf("value has type %T, not string", v) + } + + return s.genTXTAnswer(req, []string{str}), nil + default: + log.Debug("don't know how to handle dns rr type %d, skipping", rr) + + return nil, nil + } +} + +// filterDNSRewrite handles dnsrewrite filters. It constructs a DNS +// response and sets it into d.Res. +func (s *Server) filterDNSRewrite(req *dns.Msg, res dnsfilter.Result, d *proxy.DNSContext) (err error) { + resp := s.makeResponse(req) + dnsrr := res.DNSRewriteResult + if dnsrr == nil { + return agherr.Error("no dns rewrite rule content") + } + + resp.Rcode = dnsrr.RCode + if resp.Rcode != dns.RcodeSuccess { + d.Res = resp + + return nil + } + + if dnsrr.Response == nil { + return agherr.Error("no dns rewrite rule responses") + } + + rr := req.Question[0].Qtype + values := dnsrr.Response[rr] + for i, v := range values { + var ans dns.RR + ans, err = s.filterDNSRewriteResponse(req, rr, v) + if err != nil { + return fmt.Errorf("dns rewrite response for %d[%d]: %w", rr, i, err) + } + + resp.Answer = append(resp.Answer, ans) + } + + d.Res = resp + + return nil +} diff --git a/internal/dnsforward/filter.go b/internal/dnsforward/filter.go index 83effc60e07..5cd0090a987 100644 --- a/internal/dnsforward/filter.go +++ b/internal/dnsforward/filter.go @@ -42,7 +42,8 @@ func (s *Server) getClientRequestFilteringSettings(d *proxy.DNSContext) *dnsfilt return &setts } -// filterDNSRequest applies the dnsFilter and sets d.Res if the request was filtered +// filterDNSRequest applies the dnsFilter and sets d.Res if the request +// was filtered. func (s *Server) filterDNSRequest(ctx *dnsContext) (*dnsfilter.Result, error) { d := ctx.proxyCtx req := d.Req @@ -54,9 +55,13 @@ func (s *Server) filterDNSRequest(ctx *dnsContext) (*dnsfilter.Result, error) { } else if res.IsFiltered { log.Tracef("Host %s is filtered, reason - %q, matched rule: %q", host, res.Reason, res.Rules[0].Text) d.Res = s.genDNSFilterMessage(d, &res) - } else if res.Reason == dnsfilter.ReasonRewrite && len(res.CanonName) != 0 && len(res.IPList) == 0 { + } else if res.Reason.In(dnsfilter.ReasonRewrite, dnsfilter.DNSRewriteRule) && + res.CanonName != "" && + len(res.IPList) == 0 { + // Resolve the new canonical name, not the original host + // name. The original question is readded in + // processFilteringAfterResponse. ctx.origQuestion = d.Req.Question[0] - // resolve canonical name, not the original host name d.Req.Question[0].Name = dns.Fqdn(res.CanonName) } else if res.Reason == dnsfilter.RewriteAutoHosts && len(res.ReverseHosts) != 0 { resp := s.makeResponse(req) @@ -99,6 +104,11 @@ func (s *Server) filterDNSRequest(ctx *dnsContext) (*dnsfilter.Result, error) { } d.Res = resp + } else if res.Reason == dnsfilter.DNSRewriteRule { + err = s.filterDNSRewrite(req, res, d) + if err != nil { + return nil, err + } } return &res, err diff --git a/internal/dnsforward/msg.go b/internal/dnsforward/msg.go index 2e72c40c970..f8200056139 100644 --- a/internal/dnsforward/msg.go +++ b/internal/dnsforward/msg.go @@ -11,12 +11,17 @@ import ( ) // Create a DNS response by DNS request and set necessary flags -func (s *Server) makeResponse(req *dns.Msg) *dns.Msg { - resp := dns.Msg{} +func (s *Server) makeResponse(req *dns.Msg) (resp *dns.Msg) { + resp = &dns.Msg{ + MsgHdr: dns.MsgHdr{ + RecursionAvailable: true, + }, + Compress: true, + } + resp.SetReply(req) - resp.RecursionAvailable = true - resp.Compress = true - return &resp + + return resp } // genDNSFilterMessage generates a DNS message corresponding to the filtering result @@ -121,6 +126,18 @@ func (s *Server) genAAAAAnswer(req *dns.Msg, ip net.IP) *dns.AAAA { return answer } +func (s *Server) genTXTAnswer(req *dns.Msg, strs []string) (answer *dns.TXT) { + return &dns.TXT{ + Hdr: dns.RR_Header{ + Name: req.Question[0].Name, + Rrtype: dns.TypeTXT, + Ttl: s.conf.BlockedResponseTTL, + Class: dns.ClassINET, + }, + Txt: strs, + } +} + // generate DNS response message with an IP address func (s *Server) genResponseWithIP(req *dns.Msg, ip net.IP) *dns.Msg { if req.Question[0].Qtype == dns.TypeA && ip.To4() != nil { diff --git a/internal/dnsforward/stats.go b/internal/dnsforward/stats.go index c2b8921f684..c447be05c0b 100644 --- a/internal/dnsforward/stats.go +++ b/internal/dnsforward/stats.go @@ -91,7 +91,7 @@ func (s *Server) updateStats(d *proxy.DNSContext, elapsed time.Duration, res dns case dnsfilter.FilteredSafeSearch: e.Result = stats.RSafeSearch - case dnsfilter.FilteredBlackList: + case dnsfilter.FilteredBlockList: fallthrough case dnsfilter.FilteredInvalid: fallthrough diff --git a/internal/home/controlfiltering.go b/internal/home/controlfiltering.go index 3fe07e7e471..1d0172e8b59 100644 --- a/internal/home/controlfiltering.go +++ b/internal/home/controlfiltering.go @@ -359,6 +359,9 @@ type checkHostResp struct { // Deprecated: Use Rules[*].FilterListID. FilterID int64 `json:"filter_id"` + // Rule is the text of the matched rule. + // + // Deprecated: Use Rules[*].Text. Rule string `json:"rule"` Rules []*checkHostRespRule `json:"rules"` @@ -386,12 +389,15 @@ func (f *Filtering) handleCheckHost(w http.ResponseWriter, r *http.Request) { resp := checkHostResp{} resp.Reason = result.Reason.String() - resp.FilterID = result.Rules[0].FilterListID - resp.Rule = result.Rules[0].Text resp.SvcName = result.ServiceName resp.CanonName = result.CanonName resp.IPList = result.IPList + if len(result.Rules) > 0 { + resp.FilterID = result.Rules[0].FilterListID + resp.Rule = result.Rules[0].Text + } + resp.Rules = make([]*checkHostRespRule, len(result.Rules)) for i, r := range result.Rules { resp.Rules[i] = &checkHostRespRule{ diff --git a/internal/querylog/decode.go b/internal/querylog/decode.go index f59377811e6..ed7214893de 100644 --- a/internal/querylog/decode.go +++ b/internal/querylog/decode.go @@ -4,11 +4,14 @@ import ( "encoding/base64" "encoding/json" "io" + "net" "strings" "time" "github.com/AdguardTeam/AdGuardHome/internal/dnsfilter" "github.com/AdguardTeam/golibs/log" + "github.com/AdguardTeam/urlfilter/rules" + "github.com/miekg/dns" ) type logEntryHandler (func(t json.Token, ent *logEntry) error) @@ -165,13 +168,285 @@ var resultHandlers = map[string]logEntryHandler{ return nil }, "ServiceName": func(t json.Token, ent *logEntry) error { - v, ok := t.(string) + s, ok := t.(string) if !ok { return nil } - ent.Result.ServiceName = v + + ent.Result.ServiceName = s + return nil }, + "CanonName": func(t json.Token, ent *logEntry) error { + s, ok := t.(string) + if !ok { + return nil + } + + ent.Result.CanonName = s + + return nil + }, +} + +func decodeResultRuleKey(key string, i int, dec *json.Decoder, ent *logEntry) { + switch key { + case "FilterListID": + vToken, err := dec.Token() + if err != nil { + if err != io.EOF { + log.Debug("decodeResultRuleKey %s err: %s", key, err) + } + + return + } + + if len(ent.Result.Rules) < i+1 { + ent.Result.Rules = append(ent.Result.Rules, &dnsfilter.ResultRule{}) + } + + if n, ok := vToken.(json.Number); ok { + ent.Result.Rules[i].FilterListID, _ = n.Int64() + } + case "IP": + vToken, err := dec.Token() + if err != nil { + if err != io.EOF { + log.Debug("decodeResultRuleKey %s err: %s", key, err) + } + + return + } + + if len(ent.Result.Rules) < i+1 { + ent.Result.Rules = append(ent.Result.Rules, &dnsfilter.ResultRule{}) + } + + if ipStr, ok := vToken.(string); ok { + ent.Result.Rules[i].IP = net.ParseIP(ipStr) + } + case "Text": + vToken, err := dec.Token() + if err != nil { + if err != io.EOF { + log.Debug("decodeResultRuleKey %s err: %s", key, err) + } + + return + } + + if len(ent.Result.Rules) < i+1 { + ent.Result.Rules = append(ent.Result.Rules, &dnsfilter.ResultRule{}) + } + + if s, ok := vToken.(string); ok { + ent.Result.Rules[i].Text = s + } + default: + // Go on. + } +} + +func decodeResultRules(dec *json.Decoder, ent *logEntry) { + for { + delimToken, err := dec.Token() + if err != nil { + if err != io.EOF { + log.Debug("decodeResultRules err: %s", err) + } + + return + } + + if d, ok := delimToken.(json.Delim); ok { + if d != '[' { + log.Debug("decodeResultRules: unexpected delim %q", d) + } + } else { + return + } + + i := 0 + for { + keyToken, err := dec.Token() + if err != nil { + if err != io.EOF { + log.Debug("decodeResultRules err: %s", err) + } + + return + } + + if d, ok := keyToken.(json.Delim); ok { + if d == '}' { + i++ + } else if d == ']' { + return + } + + continue + } + + key, ok := keyToken.(string) + if !ok { + log.Debug("decodeResultRules: keyToken is %T (%[1]v) and not string", keyToken) + + return + } + + decodeResultRuleKey(key, i, dec, ent) + } + } +} + +func decodeResultReverseHosts(dec *json.Decoder, ent *logEntry) { + for { + itemToken, err := dec.Token() + if err != nil { + if err != io.EOF { + log.Debug("decodeResultReverseHosts err: %s", err) + } + + return + } + + switch v := itemToken.(type) { + case json.Delim: + if v == '[' { + continue + } else if v == ']' { + return + } + + log.Debug("decodeResultReverseHosts: unexpected delim %q", v) + + return + case string: + ent.Result.ReverseHosts = append(ent.Result.ReverseHosts, v) + default: + continue + } + } +} + +func decodeResultIPList(dec *json.Decoder, ent *logEntry) { + for { + itemToken, err := dec.Token() + if err != nil { + if err != io.EOF { + log.Debug("decodeResultIPList err: %s", err) + } + + return + } + + switch v := itemToken.(type) { + case json.Delim: + if v == '[' { + continue + } else if v == ']' { + return + } + + log.Debug("decodeResultIPList: unexpected delim %q", v) + + return + case string: + ip := net.ParseIP(v) + if ip != nil { + ent.Result.IPList = append(ent.Result.IPList, ip) + } + default: + continue + } + } +} + +func decodeResultDNSRewriteResult(dec *json.Decoder, ent *logEntry) { + for { + keyToken, err := dec.Token() + if err != nil { + if err != io.EOF { + log.Debug("decodeResultDNSRewriteResult err: %s", err) + } + + return + } + + if d, ok := keyToken.(json.Delim); ok { + if d == '}' { + return + } + + continue + } + + key, ok := keyToken.(string) + if !ok { + log.Debug("decodeResultDNSRewriteResult: keyToken is %T (%[1]v) and not string", keyToken) + + return + } + + // TODO(a.garipov): Refactor this into a separate + // function à la decodeResultRuleKey if we keep this + // code for a longer time than planned. + switch key { + case "RCode": + vToken, err := dec.Token() + if err != nil { + if err != io.EOF { + log.Debug("decodeResultDNSRewriteResult err: %s", err) + } + + return + } + + if ent.Result.DNSRewriteResult == nil { + ent.Result.DNSRewriteResult = &dnsfilter.DNSRewriteResult{} + } + + if n, ok := vToken.(json.Number); ok { + rcode64, _ := n.Int64() + ent.Result.DNSRewriteResult.RCode = rules.RCode(rcode64) + } + + continue + case "Response": + if ent.Result.DNSRewriteResult == nil { + ent.Result.DNSRewriteResult = &dnsfilter.DNSRewriteResult{} + } + + if ent.Result.DNSRewriteResult.Response == nil { + ent.Result.DNSRewriteResult.Response = dnsfilter.DNSRewriteResultResponse{} + } + + // TODO(a.garipov): I give up. This whole file + // is a mess. Luckily, we can assume that this + // field is relatively rare and just use the + // normal decoding and correct the values. + err = dec.Decode(&ent.Result.DNSRewriteResult.Response) + if err != nil { + log.Debug("decodeResultDNSRewriteResult response err: %s", err) + } + + for rrType, rrValues := range ent.Result.DNSRewriteResult.Response { + switch rrType { + case dns.TypeA, dns.TypeAAAA: + for i, v := range rrValues { + s, _ := v.(string) + rrValues[i] = net.ParseIP(s) + } + default: + // Go on. + } + } + + continue + default: + // Go on. + } + } } func decodeResult(dec *json.Decoder, ent *logEntry) { @@ -200,6 +475,27 @@ func decodeResult(dec *json.Decoder, ent *logEntry) { return } + switch key { + case "ReverseHosts": + decodeResultReverseHosts(dec, ent) + + continue + case "IPList": + decodeResultIPList(dec, ent) + + continue + case "Rules": + decodeResultRules(dec, ent) + + continue + case "DNSRewriteResult": + decodeResultDNSRewriteResult(dec, ent) + + continue + default: + // Go on. + } + handler, ok := resultHandlers[key] if !ok { continue diff --git a/internal/querylog/decode_test.go b/internal/querylog/decode_test.go index d9cbd6002a4..ffcf94dcd69 100644 --- a/internal/querylog/decode_test.go +++ b/internal/querylog/decode_test.go @@ -2,95 +2,181 @@ package querylog import ( "bytes" + "encoding/base64" + "net" "strings" "testing" + "time" + "github.com/AdguardTeam/AdGuardHome/internal/dnsfilter" "github.com/AdguardTeam/AdGuardHome/internal/testutil" "github.com/AdguardTeam/golibs/log" + "github.com/AdguardTeam/urlfilter/rules" + "github.com/miekg/dns" "github.com/stretchr/testify/assert" ) -func TestDecode_decodeQueryLog(t *testing.T) { +func TestDecodeLogEntry(t *testing.T) { logOutput := &bytes.Buffer{} testutil.ReplaceLogWriter(t, logOutput) testutil.ReplaceLogLevel(t, log.DEBUG) + t.Run("success", func(t *testing.T) { + const ansStr = `Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==` + const data = `{"IP":"127.0.0.1",` + + `"T":"2020-11-25T18:55:56.519796+03:00",` + + `"QH":"an.yandex.ru",` + + `"QT":"A",` + + `"QC":"IN",` + + `"CP":"",` + + `"Answer":"` + ansStr + `",` + + `"Result":{` + + `"IsFiltered":true,` + + `"Reason":3,` + + `"ReverseHosts":["example.net"],` + + `"IPList":["127.0.0.2"],` + + `"Rules":[{"FilterListID":42,"Text":"||an.yandex.ru","IP":"127.0.0.2"},` + + `{"FilterListID":43,"Text":"||an2.yandex.ru","IP":"127.0.0.3"}],` + + `"CanonName":"example.com",` + + `"ServiceName":"example.org",` + + `"DNSRewriteResult":{"RCode":0,"Response":{"1":["127.0.0.2"]}}},` + + `"Elapsed":837429}` + + ans, err := base64.StdEncoding.DecodeString(ansStr) + assert.Nil(t, err) + + want := &logEntry{ + IP: "127.0.0.1", + Time: time.Date(2020, 11, 25, 15, 55, 56, 519796000, time.UTC), + QHost: "an.yandex.ru", + QType: "A", + QClass: "IN", + ClientProto: "", + Answer: ans, + Result: dnsfilter.Result{ + IsFiltered: true, + Reason: dnsfilter.FilteredBlockList, + ReverseHosts: []string{"example.net"}, + IPList: []net.IP{net.IPv4(127, 0, 0, 2)}, + Rules: []*dnsfilter.ResultRule{{ + FilterListID: 42, + Text: "||an.yandex.ru", + IP: net.IPv4(127, 0, 0, 2), + }, { + FilterListID: 43, + Text: "||an2.yandex.ru", + IP: net.IPv4(127, 0, 0, 3), + }}, + CanonName: "example.com", + ServiceName: "example.org", + DNSRewriteResult: &dnsfilter.DNSRewriteResult{ + RCode: dns.RcodeSuccess, + Response: dnsfilter.DNSRewriteResultResponse{ + dns.TypeA: []rules.RRValue{net.IPv4(127, 0, 0, 2)}, + }, + }, + }, + Elapsed: 837429, + } + + got := &logEntry{} + decodeLogEntry(got, data) + + s := logOutput.String() + assert.Equal(t, "", s) + + // Correct for time zones. + got.Time = got.Time.UTC() + assert.Equal(t, want, got) + }) + testCases := []struct { name string log string want string }{{ - name: "all_right", - log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"Rule":"||an.yandex.","FilterID":1},"Elapsed":837429}`, - want: "default", + name: "all_right_old_rule", + log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"Rule":"||an.yandex.","FilterID":1,"ReverseHosts":["example.com"],"IPList":["127.0.0.1"]},"Elapsed":837429}`, + want: "", }, { - name: "bad_filter_id", - log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"Rule":"||an.yandex.","FilterID":1.5},"Elapsed":837429}`, + name: "bad_filter_id_old_rule", + log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"FilterID":1.5},"Elapsed":837429}`, want: "decodeResult handler err: strconv.ParseInt: parsing \"1.5\": invalid syntax\n", }, { name: "bad_is_filtered", - log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":trooe,"Reason":3,"Rule":"||an.yandex.","FilterID":1},"Elapsed":837429}`, + log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":trooe,"Reason":3},"Elapsed":837429}`, want: "decodeLogEntry err: invalid character 'o' in literal true (expecting 'u')\n", }, { name: "bad_elapsed", - log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"Rule":"||an.yandex.","FilterID":1},"Elapsed":-1}`, - want: "default", + log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3},"Elapsed":-1}`, + want: "", }, { name: "bad_ip", - log: `{"IP":127001,"T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"Rule":"||an.yandex.","FilterID":1},"Elapsed":837429}`, - want: "default", + log: `{"IP":127001,"T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3},"Elapsed":837429}`, + want: "", }, { name: "bad_time", - log: `{"IP":"127.0.0.1","T":"12/09/1998T15:00:00.000000+05:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"Rule":"||an.yandex.","FilterID":1},"Elapsed":837429}`, + log: `{"IP":"127.0.0.1","T":"12/09/1998T15:00:00.000000+05:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3},"Elapsed":837429}`, want: "decodeLogEntry handler err: parsing time \"12/09/1998T15:00:00.000000+05:00\" as \"2006-01-02T15:04:05Z07:00\": cannot parse \"9/1998T15:00:00.000000+05:00\" as \"2006\"\n", }, { name: "bad_host", - log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":6,"QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"Rule":"||an.yandex.","FilterID":1},"Elapsed":837429}`, - want: "default", + log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":6,"QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3},"Elapsed":837429}`, + want: "", }, { name: "bad_type", - log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":true,"QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"Rule":"||an.yandex.","FilterID":1},"Elapsed":837429}`, - want: "default", + log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":true,"QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3},"Elapsed":837429}`, + want: "", }, { name: "bad_class", - log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":false,"CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"Rule":"||an.yandex.","FilterID":1},"Elapsed":837429}`, - want: "default", + log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":false,"CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3},"Elapsed":837429}`, + want: "", }, { name: "bad_client_proto", - log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":8,"Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"Rule":"||an.yandex.","FilterID":1},"Elapsed":837429}`, - want: "default", + log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":8,"Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3},"Elapsed":837429}`, + want: "", }, { name: "very_bad_client_proto", - log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"dog","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"Rule":"||an.yandex.","FilterID":1},"Elapsed":837429}`, + log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"dog","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3},"Elapsed":837429}`, want: "decodeLogEntry handler err: invalid client proto: \"dog\"\n", }, { name: "bad_answer", - log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":0.9,"Result":{"IsFiltered":true,"Reason":3,"Rule":"||an.yandex.","FilterID":1},"Elapsed":837429}`, - want: "default", + log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":0.9,"Result":{"IsFiltered":true,"Reason":3},"Elapsed":837429}`, + want: "", }, { name: "very_bad_answer", - log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"Rule":"||an.yandex.","FilterID":1},"Elapsed":837429}`, + log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3},"Elapsed":837429}`, want: "decodeLogEntry handler err: illegal base64 data at input byte 61\n", }, { name: "bad_rule", - log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"Rule":false,"FilterID":1},"Elapsed":837429}`, - want: "default", + log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"Rule":false},"Elapsed":837429}`, + want: "", }, { name: "bad_reason", - log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":true,"Rule":"||an.yandex.","FilterID":1},"Elapsed":837429}`, - want: "default", + log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":true},"Elapsed":837429}`, + want: "", + }, { + name: "bad_reverse_hosts", + log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"ReverseHosts":[{}]},"Elapsed":837429}`, + want: "decodeResultReverseHosts: unexpected delim \"{\"\n", + }, { + name: "bad_ip_list", + log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"ReverseHosts":["example.net"],"IPList":[{}]},"Elapsed":837429}`, + want: "decodeResultIPList: unexpected delim \"{\"\n", }} for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - _, err := logOutput.Write([]byte("default")) - assert.Nil(t, err) - l := &logEntry{} decodeLogEntry(l, tc.log) - assert.True(t, strings.HasSuffix(logOutput.String(), tc.want), "%q\ndoes not end with\n%q", logOutput.String(), tc.want) + s := logOutput.String() + if tc.want == "" { + assert.Equal(t, "", s) + } else { + assert.True(t, strings.HasSuffix(s, tc.want), + "got %q", s) + } logOutput.Reset() }) diff --git a/internal/querylog/searchcriteria.go b/internal/querylog/searchcriteria.go index 52b76459ea4..b98e083814b 100644 --- a/internal/querylog/searchcriteria.go +++ b/internal/querylog/searchcriteria.go @@ -115,14 +115,14 @@ func (c *searchCriteria) ctFilteringStatusCase(res dnsfilter.Result) bool { case filteringStatusFiltered: return res.IsFiltered || res.Reason.In( - dnsfilter.NotFilteredWhiteList, + dnsfilter.NotFilteredAllowList, dnsfilter.ReasonRewrite, dnsfilter.RewriteAutoHosts, ) case filteringStatusBlocked: return res.IsFiltered && - res.Reason.In(dnsfilter.FilteredBlackList, dnsfilter.FilteredBlockedService) + res.Reason.In(dnsfilter.FilteredBlockList, dnsfilter.FilteredBlockedService) case filteringStatusBlockedService: return res.IsFiltered && res.Reason == dnsfilter.FilteredBlockedService @@ -134,7 +134,7 @@ func (c *searchCriteria) ctFilteringStatusCase(res dnsfilter.Result) bool { return res.IsFiltered && res.Reason == dnsfilter.FilteredSafeBrowsing case filteringStatusWhitelisted: - return res.Reason == dnsfilter.NotFilteredWhiteList + return res.Reason == dnsfilter.NotFilteredAllowList case filteringStatusRewritten: return res.Reason.In(dnsfilter.ReasonRewrite, dnsfilter.RewriteAutoHosts) @@ -144,9 +144,9 @@ func (c *searchCriteria) ctFilteringStatusCase(res dnsfilter.Result) bool { case filteringStatusProcessed: return !res.Reason.In( - dnsfilter.FilteredBlackList, + dnsfilter.FilteredBlockList, dnsfilter.FilteredBlockedService, - dnsfilter.NotFilteredWhiteList, + dnsfilter.NotFilteredAllowList, ) default: diff --git a/openapi/CHANGELOG.md b/openapi/CHANGELOG.md index f40dd49057a..076d9896e47 100644 --- a/openapi/CHANGELOG.md +++ b/openapi/CHANGELOG.md @@ -4,15 +4,21 @@ ## v0.105: API changes -### Multiple matched rules in `GET /filtering/check_host` and `GET /querylog` +### New `"reason"` in `GET /filtering/check_host` and `GET /querylog` + +* The new `DNSRewriteRule` reason is added to `GET /filtering/check_host` and + `GET /querylog`. - +* Also, the reason which was incorrectly documented as `"ReasonRewrite"` is now + correctly documented as `"Rewrite"`, and the previously undocumented + `"RewriteEtcHosts"` is now documented as well. + +### Multiple matched rules in `GET /filtering/check_host` and `GET /querylog` * The properties `rule` and `filter_id` are now deprecated. API users should - inspect the newly-added `rules` object array instead. Currently, it's either - empty or contains one object, which contains the same things as the old two - properties did, but under more correct names: + inspect the newly-added `rules` object array instead. For most rules, it's + either empty or contains one object, which contains the same things as the old + two properties did, but under more correct names: ```js { @@ -30,6 +36,30 @@ checked in. --> } ``` + For `$dnsrewrite` rules, they contain all rules that contributed to the + result. For example, if you have the following filtering rules: + + ``` + ||example.com^$dnsrewrite=127.0.0.1 + ||example.com^$dnsrewrite=127.0.0.2 + ``` + + The `"rules"` will be something like: + + ```js + { + // … + + "rules": [{ + "text": "||example.com^$dnsrewrite=127.0.0.1", + "filter_list_id": 0 + }, { + "text": "||example.com^$dnsrewrite=127.0.0.2", + "filter_list_id": 0 + }] + } + ``` + The old fields will be removed in v0.106.0. ## v0.103: API changes diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index 17e9f3f529b..61db836e43b 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -523,7 +523,7 @@ Reload filtering rules from URLs. This might be needed if new URL was just added and you dont want to wait for automatic refresh to kick in. This API request is ratelimited, so you can call it freely as often as - you like, it wont create unneccessary burden on servers that host the + you like, it wont create unnecessary burden on servers that host the URL. This should work as intended, a `force` parameter is offered as last-resort attempt to make filter lists fresh. If you ever find yourself using `force` to make something work that otherwise wont, this @@ -1246,7 +1246,7 @@ 'properties': 'reason': 'type': 'string' - 'description': 'DNS filter status' + 'description': 'Request filtering status.' 'enum': - 'NotFilteredNotFound' - 'NotFilteredWhiteList' @@ -1257,7 +1257,9 @@ - 'FilteredInvalid' - 'FilteredSafeSearch' - 'FilteredBlockedService' - - 'ReasonRewrite' + - 'Rewrite' + - 'RewriteEtcHosts' + - 'DNSRewriteRule' 'filter_id': 'deprecated': true 'description': > @@ -1284,12 +1286,12 @@ 'description': 'Set if reason=FilteredBlockedService' 'cname': 'type': 'string' - 'description': 'Set if reason=ReasonRewrite' + 'description': 'Set if reason=Rewrite' 'ip_addrs': 'type': 'array' 'items': 'type': 'string' - 'description': 'Set if reason=ReasonRewrite' + 'description': 'Set if reason=Rewrite' 'FilterRefreshResponse': 'type': 'object' 'description': '/filtering/refresh response data' @@ -1648,7 +1650,7 @@ '$ref': '#/components/schemas/ResultRule' 'reason': 'type': 'string' - 'description': 'DNS filter status' + 'description': 'Request filtering status.' 'enum': - 'NotFilteredNotFound' - 'NotFilteredWhiteList' @@ -1659,7 +1661,9 @@ - 'FilteredInvalid' - 'FilteredSafeSearch' - 'FilteredBlockedService' - - 'ReasonRewrite' + - 'Rewrite' + - 'RewriteEtcHosts' + - 'DNSRewriteRule' 'service_name': 'type': 'string' 'description': 'Set if reason=FilteredBlockedService' diff --git a/scripts/go-lint.sh b/scripts/go-lint.sh index c133379b60e..ca30e1e78e3 100644 --- a/scripts/go-lint.sh +++ b/scripts/go-lint.sh @@ -95,7 +95,7 @@ ineffassign . unparam ./... -misspell --error ./... +git ls-files -- '*.go' '*.md' '*.yaml' '*.yml' | xargs misspell --error looppointer ./... diff --git a/scripts/translations/README.md b/scripts/translations/README.md index 2c8e56363b2..3a5e336cda5 100644 --- a/scripts/translations/README.md +++ b/scripts/translations/README.md @@ -1,4 +1,4 @@ -## Twosky intergration script +## Twosky integration script ### Usage