Allow users to specify name-servers in input regardless of nameserver…

…s/local addresses supplied with flags (#437)

* examples pass and code compiles

* fixed up modules

* module tests

* module tests working

* forgot to commit modules

* unit tests passing

* handle empty nameserver correctly

* added new unit tests

* added two negative test cases

* added integration test for new domain nameserver feature

* lint

* fixed (hopefully) host IP capability detection

* fix for metadata integration test when running in parrallel

* make file names unique for test parallelism

* use zdns defaults for name-server-mode

* review

* added integration tests

* create connInfo objs. on demand

* unit tests passing

* fixed bool logic bug

* fixed up integration tests

* lowered core count for make test, was seeing timeout issues when running tests on macbook

* remove local addr verification checks

* error if user specifies IPv4 and no nameservers are available, vice versa for IPv6

* remove unneeded google addresses

* removed loopback ipv6 test

* cleared up NSLookup msg if IP-lookup not specified

makefile

@@ -12,7 +12,7 @@ install: zdns
 test: zdns
 	go test -v ./...
 	pip3 install -r testing/requirements.txt
-	pytest -n auto testing/integration_tests.py
+	pytest -n 4 testing/integration_tests.py
 
 integration-tests: zdns
 	pip3 install -r testing/requirements.txt

src/cli/worker_manager.go

@@ -38,12 +38,6 @@ import (
 	"github.com/zmap/zdns/src/zdns"
 )
 
-const (
-	googleDNSResolverAddr   = "8.8.8.8:53"
-	googleDNSResolverAddrV6 = "[2001:4860:4860::8888]:53"
-	loopbackIPv4Addr        = "127.0.0.1"
-)
-
 type routineMetadata struct {
 	Names   int // number of domain names processed
 	Lookups int // number of lookups performed
@@ -236,14 +230,12 @@ func populateResolverConfig(gc *CLIConf) *zdns.ResolverConfig {
 		config.RootNameServersV4 = []zdns.NameServer{}
 	}
 	noV4NameServers := len(config.ExternalNameServersV4) == 0 && len(config.RootNameServersV4) == 0
-	if config.IPVersionMode != zdns.IPv6Only && noV4NameServers {
-		log.Info("no IPv4 nameservers found. Switching to --6 only")
-		config.IPVersionMode = zdns.IPv6Only
+	if gc.IPv4TransportOnly && noV4NameServers {
+		log.Fatal("cannot use --4 since no IPv4 nameservers found, ensure you have IPv4 connectivity and provide --name-servers")
 	}
 	noV6NameServers := len(config.ExternalNameServersV6) == 0 && len(config.RootNameServersV6) == 0
-	if config.IPVersionMode != zdns.IPv4Only && noV6NameServers {
-		log.Info("no IPv6 nameservers found. Switching to --4 only")
-		config.IPVersionMode = zdns.IPv4Only
+	if gc.IPv6TransportOnly && noV6NameServers {
+		log.Fatal("cannot use --6 since no IPv6 nameservers found, ensure you have IPv6 connectivity and provide --name-servers")
 	}
 
 	config, err = populateLocalAddresses(gc, config)
@@ -255,7 +247,9 @@ func populateResolverConfig(gc *CLIConf) *zdns.ResolverConfig {
 
 // populateIPTransportMode populates the IPTransportMode field of the ResolverConfig
 // If user sets --4 (IPv4 Only) or --6 (IPv6 Only), we'll set the IPVersionMode to IPv4Only or IPv6Only, respectively.
-// Otherwise, we need to determine the IPVersionMode based on either the OS' default resolver(s) or the user's provided name servers.
+// If user does not set --4 or --6, we'll determine the IPVersionMode based on:
+//  1. the provided name-servers (if any)
+//  2. the OS' default resolvers (if no name-servers provided)
 func populateIPTransportMode(gc *CLIConf, config *zdns.ResolverConfig) (*zdns.ResolverConfig, error) {
 	if gc.IPv4TransportOnly && gc.IPv6TransportOnly {
 		return nil, errors.New("only one of --4 and --6 allowed")
@@ -387,10 +381,8 @@ func populateNameServers(gc *CLIConf, config *zdns.ResolverConfig) (*zdns.Resolv
 func populateLocalAddresses(gc *CLIConf, config *zdns.ResolverConfig) (*zdns.ResolverConfig, error) {
 	// Local Addresses are populated in this order:
 	// 1. If user provided local addresses, use those
-	// 2. If the config's nameservers are loopback, use the local loopback address
-	// 3. Otherwise, try to connect to Google's recursive resolver and take the IP address we use for the connection
+	// 2. If user does not provide local addresses, one will be used on-demand by Resolver. See resolver.go:getConnectionInfo for more info
 
-	// IPv4 local addresses are required for IPv4 lookups, same for IPv6
 	if len(gc.LocalAddrs) != 0 {
 		// if user provided a local address(es), that takes precedent
 		config.LocalAddrsV4, config.LocalAddrsV6 = []net.IP{}, []net.IP{}
@@ -406,50 +398,6 @@ func populateLocalAddresses(gc *CLIConf, config *zdns.ResolverConfig) (*zdns.Res
 				return nil, fmt.Errorf("invalid local address: %s", addr.String())
 			}
 		}
-		return config, nil
-	}
-	// if the nameservers are loopback, use the loopback address
-	allNameServers := util.Concat(config.ExternalNameServersV4, config.ExternalNameServersV6, config.RootNameServersV4, config.RootNameServersV6)
-	if len(allNameServers) == 0 {
-		// this shouldn't happen
-		return nil, errors.New("name servers must be set before populating local addresses")
-	}
-	anyNameServersLoopack := false
-	for _, ns := range allNameServers {
-		if ns.IP.IsLoopback() {
-			anyNameServersLoopack = true
-			break
-		}
-	}
-
-	if anyNameServersLoopack {
-		// set local address so name servers are reachable
-		config.LocalAddrsV4 = []net.IP{net.ParseIP(loopbackIPv4Addr)}
-		// loopback nameservers not supported for IPv6, we'll let Resolver validation take care of this
-	} else {
-		// localAddr not set, so we need to find the default IP address
-		if config.IPVersionMode != zdns.IPv6Only {
-			conn, err := net.Dial("udp", googleDNSResolverAddr)
-			if err != nil {
-				return nil, fmt.Errorf("unable to find default IP address to open socket: %w", err)
-			}
-			config.LocalAddrsV4 = []net.IP{conn.LocalAddr().(*net.UDPAddr).IP}
-			// cleanup socket
-			if err = conn.Close(); err != nil {
-				log.Error("unable to close test connection to Google public DNS: ", err)
-			}
-		}
-		if config.IPVersionMode != zdns.IPv4Only {
-			conn, err := net.Dial("udp", googleDNSResolverAddrV6)
-			if err != nil {
-				return nil, fmt.Errorf("unable to find default IP address to open socket: %w", err)
-			}
-			config.LocalAddrsV6 = []net.IP{conn.LocalAddr().(*net.UDPAddr).IP}
-			// cleanup socket
-			if err = conn.Close(); err != nil {
-				log.Error("unable to close test connection to Google public IPv6 DNS: ", err)
-			}
-		}
 	}
 	return config, nil
 }

src/modules/nslookup/ns_lookup.go

@@ -38,7 +38,7 @@ type NSLookupModule struct {
 // CLIInit initializes the NSLookupModule with the given parameters, used to call NSLookup from the command line
 func (nsMod *NSLookupModule) CLIInit(gc *cli.CLIConf, resolverConf *zdns.ResolverConfig) error {
 	if !nsMod.IPv4Lookup && !nsMod.IPv6Lookup {
-		log.Debug("NSModule: No IP version specified, defaulting to IPv4")
+		log.Debug("NSModule: neither --ipv4-lookup nor --ipv6-lookup specified, will only request A records for each NS server")
 		nsMod.IPv4Lookup = true
 	}
 	err := nsMod.BasicLookupModule.CLIInit(gc, resolverConf)

src/zdns/lookup.go

@@ -427,22 +427,14 @@ func (r *Resolver) retryingLookup(ctx context.Context, q Question, nameServer *N
 	if nameServer == nil {
 		return SingleQueryResult{}, StatusIllegalInput, 0, errors.New("no nameserver specified")
 	}
-	var connInfo *ConnectionInfo
-	if nameServer.IP.To4() != nil {
-		connInfo = r.connInfoIPv4
-	} else if util.IsIPv6(&nameServer.IP) {
-		connInfo = r.connInfoIPv6
-	} else {
-		return SingleQueryResult{}, StatusError, 0, fmt.Errorf("could not determine IP version of nameserver: %s", nameServer)
+	connInfo, err := r.getConnectionInfo(nameServer)
+	if err != nil {
+		return SingleQueryResult{}, StatusError, 0, fmt.Errorf("could not get a connection info to query nameserver %s: %v", nameServer, err)
 	}
 	// check that our connection info is valid
 	if connInfo == nil {
 		return SingleQueryResult{}, StatusError, 0, fmt.Errorf("no connection info for nameserver: %s", nameServer)
 	}
-	// check loopback consistency
-	if nameServer.IP.IsLoopback() != connInfo.localAddr.IsLoopback() {
-		return SingleQueryResult{}, StatusIllegalInput, 0, fmt.Errorf("nameserver %s must be reachable from the local address %s, ie. both must be loopback or not loopback", nameServer, connInfo.localAddr.String())
-	}
 	r.verboseLog(1, "****WIRE LOOKUP*** ", dns.TypeToString[q.Type], " ", q.Name, " ", nameServer)
 	for i := 0; i <= r.retries; i++ {
 		// check context before going into wireLookup

src/zdns/lookup_test.go

@@ -1980,13 +1980,6 @@ func TestInvalidInputsLookup(t *testing.T) {
 		_, _, _, err := resolver.ExternalLookup(&q, &NameServer{IP: net.ParseIP("127.0.0.53")})
 		assert.Nil(t, err)
 	})
-	t.Run("using a loopback local addr with non-loopback nameserver", func(t *testing.T) {
-		result, trace, status, err := resolver.ExternalLookup(&q, &NameServer{IP: net.ParseIP("1.1.1.1"), Port: 53})
-		assert.Nil(t, result)
-		assert.Nil(t, trace)
-		assert.Equal(t, StatusIllegalInput, status)
-		assert.NotNil(t, err)
-	})
 	t.Run("invalid nameserver address", func(t *testing.T) {
 		result, trace, status, err := resolver.ExternalLookup(&q, &NameServer{IP: net.ParseIP("987.987.987.987"), Port: 53})
 		assert.Nil(t, result)