add dns os cache clearing to avoid cached high ttl negative responses

pkg/kwt/cmd/net/dns_factory.go

@@ -37,6 +37,8 @@ func (f DNSServerFactory) NewDNSServer(dstConnFactory dstconn.Factory) (ctlnet.D
 		return nil, err
 	}
 
+	opts.DomainsChangedFunc = ctlnet.NewDNSOSCache(f.logger).Flush
+
 	server, err := ctldns.NewFactory().Build(opts, f.logger)
 	if err != nil {
 		return nil, fmt.Errorf("Building server: %s", err)
@@ -50,19 +52,48 @@ func (f DNSServerFactory) NewDNSServer(dstConnFactory dstconn.Factory) (ctlnet.D
 	return server, nil
 }
 
+func (f DNSServerFactory) NewDNSOSCache() ctlnet.DNSOSCache {
+	return ctlnet.NewDNSOSCache(f.logger)
+}
+
 func (f DNSServerFactory) buildServerOpts(kubeIPResolver ctldns.IPResolver) (ctldns.BuildOpts, error) {
+	domainsMap := map[string]ctldns.IPResolver{}
+
+	for _, val := range f.dnsFlags.Map {
+		pieces := strings.SplitN(val, "=", 2)
+		if len(pieces) != 2 {
+			return ctldns.BuildOpts{}, fmt.Errorf("Expected domain to IP mapping to be in format 'domain=ip' but was '%s'", val)
+		}
+
+		ip := net.ParseIP(pieces[1])
+		if ip == nil {
+			return ctldns.BuildOpts{}, fmt.Errorf("Expected domain to IP mapping to have valid IP '%s'", val)
+		}
+
+		domainsMap[pieces[0]] = ctldns.NewStaticIPsResolver([]net.IP{ip})
+	}
+
 	opts := ctldns.BuildOpts{
 		ListenAddrs:   []string{"localhost:0"},
 		RecursorAddrs: f.dnsFlags.Recursors,
 
-		Domains: map[string]ctldns.IPResolver{
+		DomainsMapFunc: func() (map[string]ctldns.IPResolver, error) {
+			result, err := DomainsMapExecs{f.dnsFlags.MapExecs}.Get()
+			if err != nil {
+				return result, err
+			}
+
+			for domain, resolver := range domainsMap {
+				result[domain] = resolver
+			}
+
 			// Add mdns domain to regular resolver since some programs
 			// may just use /etc/resolv.conf for DNS resolution on OS X (eg dig)
 			// instead of relying on standard OS X resolution libraries
-			ctlmdns.Domain: kubeIPResolver,
-		},
+			result[ctlmdns.Domain] = kubeIPResolver
 
-		DomainsFunc: DomainsMapExecs{f.dnsFlags.MapExecs}.Get,
+			return result, nil
+		},
 	}
 
 	if len(opts.RecursorAddrs) == 0 {
@@ -81,20 +112,6 @@ func (f DNSServerFactory) buildServerOpts(kubeIPResolver ctldns.IPResolver) (ctl
 		}
 	}
 
-	for _, val := range f.dnsFlags.Map {
-		pieces := strings.SplitN(val, "=", 2)
-		if len(pieces) != 2 {
-			return ctldns.BuildOpts{}, fmt.Errorf("Expected domain to IP mapping to be in format 'domain=ip' but was '%s'", val)
-		}
-
-		ip := net.ParseIP(pieces[1])
-		if ip == nil {
-			return ctldns.BuildOpts{}, fmt.Errorf("Expected domain to IP mapping to have valid IP '%s'", val)
-		}
-
-		opts.Domains[pieces[0]] = ctldns.NewStaticIPsResolver([]net.IP{ip})
-	}
-
 	return opts, nil
 }
 

pkg/kwt/dns/domains_mux.go

@@ -1,16 +1,22 @@
 package dns
 
 import (
+	"fmt"
 	"time"
 
 	"github.com/miekg/dns"
 )
 
-type DomainsFunc func() (map[string]IPResolver, error)
+type (
+	DomainsMapFunc     func() (map[string]IPResolver, error)
+	DomainsChangedFunc func()
+)
 
 type DomainsMux struct {
-	mux         *dns.ServeMux
-	domainsFunc DomainsFunc
+	mux *dns.ServeMux
+
+	mapFunc     DomainsMapFunc
+	changedFunc DomainsChangedFunc
 	prevDomains map[string]struct{}
 
 	logTag string
@@ -19,44 +25,49 @@ type DomainsMux struct {
 
 var _ dns.Handler = &DomainsMux{}
 
-func NewDomainsMux(mux *dns.ServeMux, domainsFunc DomainsFunc, logger Logger) *DomainsMux {
-	return &DomainsMux{mux, domainsFunc, map[string]struct{}{}, "dns.DomainsMux", logger}
+func NewDomainsMux(mux *dns.ServeMux, mapFunc DomainsMapFunc, changedFunc DomainsChangedFunc, logger Logger) *DomainsMux {
+	return &DomainsMux{mux, mapFunc, changedFunc, map[string]struct{}{}, "dns.DomainsMux", logger}
 }
 
 func (m *DomainsMux) ServeDNS(w dns.ResponseWriter, r *dns.Msg) {
 	m.mux.ServeDNS(w, r)
 }
 
-func (m *DomainsMux) UpdateDomainsContiniously() {
+// UpdateContiniously and UpdateOnce are not thread safe
+func (m *DomainsMux) UpdateContiniously() {
 	for {
-		err := m.updateDomainsOnce()
+		err := m.UpdateOnce()
 		if err != nil {
 			m.logger.Debug(m.logTag, "Failed updating DNS domain handlers: %s", err)
 		}
 		time.Sleep(30 * time.Second)
 	}
 }
 
-func (m *DomainsMux) updateDomainsOnce() error {
-	domains, err := m.domainsFunc()
+func (m *DomainsMux) UpdateOnce() error {
+	domains, err := m.mapFunc()
 	if err != nil {
-		return err
+		return fmt.Errorf("Fetching domains: %s", err)
 	}
 
 	m.logger.Debug(m.logTag, "Updating DNS domain handlers: %v", domains)
 
+	changed := false
+
 	for domain, resolver := range domains {
 		if _, found := m.prevDomains[domain]; found {
 			delete(m.prevDomains, domain)
 		} else {
 			m.logger.Info(m.logTag, "Registering %s->%s", domain, resolver)
+			changed = true
 		}
 		m.mux.Handle(domain, NewCustomHandler(resolver, m.logger))
 	}
 
 	// Delete previously registered handlers that were not replaced
 	for domain, _ := range m.prevDomains {
 		m.logger.Info(m.logTag, "Unregistering %s", domain)
+		changed = true
 		m.mux.HandleRemove(domain)
 	}
 
@@ -67,5 +78,9 @@ func (m *DomainsMux) updateDomainsOnce() error {
 		m.prevDomains[domain] = struct{}{}
 	}
 
+	if changed {
+		m.changedFunc()
+	}
+
 	return nil
 }

pkg/kwt/dns/factory.go

@@ -7,10 +7,11 @@ import (
 type Factory struct{}
 
 type BuildOpts struct {
-	ListenAddrs   []string              // include port
-	RecursorAddrs []string              // include port
-	Domains       map[string]IPResolver // example "test."
-	DomainsFunc   DomainsFunc
+	ListenAddrs   []string // include port
+	RecursorAddrs []string // include port
+
+	DomainsMapFunc     DomainsMapFunc
+	DomainsChangedFunc DomainsChangedFunc
 }
 
 func NewFactory() Factory { return Factory{} }
@@ -21,14 +22,17 @@ func (f Factory) Build(opts BuildOpts, logger Logger) (Server, error) {
 	arpaHandler := NewArpaHandler(forwardHandler, logger)
 
 	mux := dns.NewServeMux()
-	for domain, resolver := range opts.Domains {
-		mux.Handle(dns.Fqdn(domain), NewCustomHandler(resolver, logger))
-	}
 	mux.Handle("arpa.", arpaHandler)
 	mux.Handle(".", forwardHandler)
 
-	domainsMux := NewDomainsMux(mux, opts.DomainsFunc, logger)
-	go domainsMux.UpdateDomainsContiniously()
+	domainsMux := NewDomainsMux(mux, opts.DomainsMapFunc, opts.DomainsChangedFunc, logger)
+
+	err := domainsMux.UpdateOnce()
+	if err != nil {
+		return Server{}, err
+	}
+
+	go domainsMux.UpdateContiniously()
 
 	servers := []*dns.Server{}
 

pkg/kwt/kubedns/kubedns_ip_resolver.go

@@ -29,6 +29,8 @@ func NewKubeDNSIPResolver(coreClient kubernetes.Interface) KubeDNSIPResolver {
 	return KubeDNSIPResolver{coreClient}
 }
 
+func (r KubeDNSIPResolver) String() string { return "kube-dns" }
+
 func (r KubeDNSIPResolver) ResolveIPv4(question string) ([]net.IP, bool, error) {
 	if !strings.HasSuffix(question, clusterSuffix) {
 		return nil, false, nil

pkg/kwt/net/dns_os_cache.go

@@ -0,0 +1,57 @@
+package net
+
+import (
+	"os/exec"
+	"runtime"
+)
+
+// DNSOSCache represents DNS caching system that Operating System configures.
+type DNSOSCache struct {
+	logger Logger
+	logTag string
+}
+
+func NewDNSOSCache(logger Logger) DNSOSCache {
+	return DNSOSCache{logger, "dns.DNSOSCache"}
+}
+
+func (c DNSOSCache) Flush() {
+	switch runtime.GOOS {
+	case "darwin":
+		// Most OS Xs have mDNSResponder which caches entries going thru native DNS resolution.
+		// If cache isnt cleared before our own DNS resolution takes over, following case may
+		// happen (inability to resolve addresses that were "negatively" cached):
+		// - before starting kwt, resolve 'foo.test'
+		//   - mDNSResponder will cache negative result with a very high TTL
+		//     because foo.test isnt typically resolvable
+		// - start kwt net start --dns-map test=127.0.0.1
+		// - resolve 'foo.test' again, expecting 127.0.0.1
+		//   - via dig it works because it bypasses OS X resolution
+		//   - via curl it does not work since negative result is still cached by OS X
+		// See mDNSResponder's internal cache via:
+		// $ log stream --predicate 'process == "mDNSResponder"' --info
+		// $ sudo killall -INFO mDNSResponder
+		c.flushOSX()
+
+	default:
+		c.logger.Debug(c.logTag, "Skipping clearing of OS DNS cache")
+	}
+}
+
+func (c DNSOSCache) flushOSX() {
+	out, err := exec.Command("killall", "-HUP", "mDNSResponder").CombinedOutput()
+	if err != nil {
+		c.logger.Debug(c.logTag, "Failed clearing mDNSResponder cache: %s (output: %s)", err, out)
+	} else {
+		c.logger.Debug(c.logTag, "Successfully cleared via mDNSResponder")
+		return
+	}
+
+	// Try flushing Directory Service cache which may on some versions of OS X do the trick
+	out, err = exec.Command("discoveryutil", "udnsflushcaches").CombinedOutput()
+	if err != nil {
+		c.logger.Debug(c.logTag, "Failed clearing via discoveryutil: %s (output: %s)", err, out)
+	} else {
+		c.logger.Debug(c.logTag, "Successfully cleared via discoveryutil")
+	}
+}

pkg/kwt/net/forwarding_proxy.go

@@ -86,6 +86,8 @@ func (o *ForwardingProxy) Serve(dstConnFactory dstconn.Factory, subnets []net.IP
 			return
 		}
 
+		o.dnsServerFactory.NewDNSOSCache().Flush()
+
 		o.logger.Info(o.logTag, "Ready!")
 	}()
 

pkg/kwt/net/interfaces.go

@@ -21,6 +21,7 @@ type DNSIPs interface {
 
 type DNSServerFactory interface {
 	NewDNSServer(dstconn.Factory) (DNSServer, error)
+	NewDNSOSCache() DNSOSCache
 }
 
 type DNSServer interface {