Add support for IPv6 (#410)

* handled ipv6 nses in the ResolverConfig

* added v6 local addrs to config

* added most basic IPv6 NS test

* added comments explaining we'll mix IPv4 and IPv6 addresses in the CLI side and distinguish later

* CLI side done

* init LocalAddr arrays in new RC and return error if ipv6only and we can't get an IPv6 address

* compiles with IPv6 support

* remove ipv6 todo

* fix bug in parsing IPv6 resolv.conf

* populate root servers with IPv6 if applicable

* fixed population bug with ipv6 roots

* fixed seg fault with a guard condition, need to fix root cause tho

* added thread-saftey for PopulateAndValidate and fixed some lack of IPv6 support in authority/additional iteration

* added a few sanity tests for IPv6

* fixed bug by not deleting unneeded local addrs and nameservers

* fixed executable name for testing

* loopback handling in ExternalLoopback

* fixed compile issues in tests

* fixed bug with proper IPv6 detection and make populateAndValidate idempotent

* fixed issue with copying ns arrays in resolver init

* added new --4 and --6 flags to disambiguate from the lookup A and AAAA flags

* fix up unit tests, messed them up by misunderstanding the lookup-ipv4 flag

* fix up IPv6 test so it doesn't run on non-IPv6 supported hosts

* forgot to rename the ZDNS exe back

* moved ipv6 tests into their own workflow

* renamed zdns exe

* cleaned up ipv6 tests

* added loopback test for ipv6

* elevate warning msg about not being able to find IPv6 socket to warning or else it's hard to identify

* add fix for using root servers if in iterative mode in CLI

* added details to loopback warning msg

* lookup AAAA for nameservers in extract authority

* added prefer ipv4 and ipv6 options

* better handling around the ipv4 preference validation, better UX, more tests

* lint

* hack to get IPv6 test to pass on hosts that don't support IPv6

* a lil more hack

* added comment to explain hack in unit tests

* fixed compile issues in non-test code

* tests passing

* lint

* remove tests with loopback IPv6, we can't have IPv6 loopback NSs

* only add name servers if IP mode supports it

* cleanup and fixed some bugs from merge

* loopback cli hack and added more string sanitization on listing --name-servers

* don't overwrite cli provided external NS'

* don't overwrite cli provided external NS bug

* compiling

* fixed up tests and added null checks to ExternalLookup. Removed test trying to use IPv6 loopback nameserver, since we don't support that

* cleaned up unneeded changes in integration_tests

* spelling

* use new concat

* avoid redundent check and remove todo

* Stop handling a domain if all nameservers don't provide sufficient glue records when they should (#417)

* stop handling a domain if the nameserver that should provide glue records doesn't

* add rfc comment

* lint

* updated ipv6 integration test

* update comment

* better err msg if user specifies IPv6 mode on non IPv6 capable machine

* infer IP support thru nameservers, use --4/6 as IPvX only

* disallow both --4 and --6

* tests and lints

* fixed up ipv6 tests

makefile

@@ -17,6 +17,11 @@ integration-tests: zdns
 	python3 testing/integration_tests.py
 	python3 testing/large_scan_integration/large_scan_integration_tests.py
 
+# Not all hosts support this, so this will be a custom make target
+ipv6-tests: zdns
+	pip3 install -r testing/requirements.txt
+	python3 testing/ipv6_tests.py
+
 lint:
 	goimports -w -local "github.com/zmap/zdns" ./
 	gofmt -s -w ./

src/cli/alookup.go

@@ -11,6 +11,7 @@
  * implied. See the License for the specific language governing
  * permissions and limitations under the License.
  */
+
 package cli
 
 import (
@@ -40,8 +41,8 @@ Specifically, alookup acts similar to nslookup and will follow CNAME records.`,
 func init() {
 	rootCmd.AddCommand(alookupCmd)
 
-	alookupCmd.PersistentFlags().Bool("ipv4-lookup", false, "perform A lookups for each MX server")
-	alookupCmd.PersistentFlags().Bool("ipv6-lookup", false, "perform AAAA record lookups for each MX server")
+	alookupCmd.PersistentFlags().Bool("ipv4-lookup", false, "perform A lookups for each server")
+	alookupCmd.PersistentFlags().Bool("ipv6-lookup", false, "perform AAAA record lookups for each server")
 
 	util.BindFlags(alookupCmd, viper.GetViper(), util.EnvPrefix)
 }

src/cli/cli.go

@@ -69,6 +69,10 @@ type CLIConf struct {
 	LookupAllNameServers bool
 	TCPOnly              bool
 	UDPOnly              bool
+	IPv4TransportOnly    bool // IPv4 transport only, incompatible with IPv6 transport only
+	IPv6TransportOnly    bool // IPv6 transport only, incompatible with IPv4 transport only
+	PreferIPv4Iteration  bool // Prefer IPv4/A record lookups during iterative resolution, only used if both IPv4 and IPv6 transport are enabled
+	PreferIPv6Iteration  bool // Prefer IPv6/AAAA record lookups during iterative resolution, only used if both IPv4 and IPv6 transport are enabled
 	RecycleSockets       bool
 	LocalAddrSpecified   bool
 	LocalAddrs           []net.IP
@@ -165,6 +169,11 @@ func init() {
 	rootCmd.PersistentFlags().StringVar(&GC.NameServersString, "name-servers", "", "List of DNS servers to use. Can be passed as comma-delimited string or via @/path/to/file. If no port is specified, defaults to 53.")
 	rootCmd.PersistentFlags().StringVar(&GC.LocalAddrString, "local-addr", "", "comma-delimited list of local addresses to use, serve as the source IP for outbound queries")
 	rootCmd.PersistentFlags().StringVar(&GC.LocalIfaceString, "local-interface", "", "local interface to use")
+	rootCmd.PersistentFlags().BoolVar(&GC.IPv4TransportOnly, "4", false, "utilize IPv4 query transport only, incompatible with --6")
+	rootCmd.PersistentFlags().BoolVar(&GC.IPv6TransportOnly, "6", false, "utilize IPv6 query transport only, incompatible with --4")
+	rootCmd.PersistentFlags().BoolVar(&GC.PreferIPv4Iteration, "prefer-ipv4-iteration", false, "Prefer IPv4/A record lookups during iterative resolution. Ignored unless used with both IPv4 and IPv6")
+	rootCmd.PersistentFlags().BoolVar(&GC.PreferIPv6Iteration, "prefer-ipv6-iteration", false, "Prefer IPv6/AAAA record lookups during iterative resolution. Ignored unless used with both IPv4 and IPv6")
+
 	rootCmd.PersistentFlags().StringVar(&GC.ConfigFilePath, "conf-file", zdns.DefaultNameServerConfigFile, "config file for DNS servers")
 	rootCmd.PersistentFlags().IntVar(&GC.Timeout, "timeout", 15, "timeout for resolving a individual name, in seconds")
 	rootCmd.PersistentFlags().IntVar(&GC.IterationTimeout, "iteration-timeout", 4, "timeout for a single iterative step in an iterative query, in seconds. Only applicable with --iterative")
@@ -174,8 +183,8 @@ func init() {
 	rootCmd.PersistentFlags().BoolVar(&GC.Dnssec, "dnssec", false, "Requests DNSSEC records by setting the DNSSEC OK (DO) bit")
 	rootCmd.PersistentFlags().BoolVar(&GC.UseNSID, "nsid", false, "Request NSID.")
 
-	rootCmd.PersistentFlags().Bool("ipv4-lookup", false, "Perform an IPv4 Lookup in modules")
-	rootCmd.PersistentFlags().Bool("ipv6-lookup", false, "Perform an IPv6 Lookup in modules")
+	rootCmd.PersistentFlags().Bool("ipv4-lookup", false, "Perform an IPv4 Lookup (requests A records) in modules")
+	rootCmd.PersistentFlags().Bool("ipv6-lookup", false, "Perform an IPv6 Lookup (requests AAAA recoreds) in modules")
 	rootCmd.PersistentFlags().StringVar(&GC.BlacklistFilePath, "blacklist-file", "", "blacklist file for servers to exclude from lookups")
 }
 

src/cli/config_validation.go

@@ -39,6 +39,7 @@ func populateNetworkingConfig(gc *CLIConf) error {
 		return errors.Wrap(err, "client subnet did not pass validation")
 	}
 
+	// local address - the user can enter both IPv4 and IPv6 addresses. We'll differentiate them later
 	if GC.LocalAddrString != "" {
 		for _, la := range strings.Split(GC.LocalAddrString, ",") {
 			ip := net.ParseIP(la)
@@ -51,6 +52,7 @@ func populateNetworkingConfig(gc *CLIConf) error {
 		gc.LocalAddrSpecified = true
 	}
 
+	// local interface - same as local addresses, an interface could have both IPv4 and IPv6 addresses, we'll differentiate them later
 	if gc.LocalIfaceString != "" {
 		li, err := net.InterfaceByName(gc.LocalIfaceString)
 		if err != nil {
@@ -114,7 +116,7 @@ func parseNameServers(gc *CLIConf) error {
 		if gc.NameServerMode {
 			log.Fatal("name servers cannot be specified on command line in --name-server-mode")
 		}
-		var ns []string
+		var nses []string
 		if (gc.NameServersString)[0] == '@' {
 			filepath := (gc.NameServersString)[1:]
 			f, err := os.ReadFile(filepath)
@@ -124,11 +126,16 @@ func parseNameServers(gc *CLIConf) error {
 			if len(f) == 0 {
 				log.Fatalf("Empty file (%s)", filepath)
 			}
-			ns = strings.Split(strings.Trim(string(f), "\n"), "\n")
+			nses = strings.Split(strings.Trim(string(f), "\n"), "\n")
 		} else {
-			ns = strings.Split(gc.NameServersString, ",")
+			nses = strings.Split(gc.NameServersString, ",")
+			trimmedNSes := make([]string, 0, len(nses))
+			for _, ns := range nses {
+				trimmedNSes = append(trimmedNSes, strings.TrimSpace(ns))
+			}
+			nses = trimmedNSes
 		}
-		gc.NameServers = ns
+		gc.NameServers = nses
 	}
 	return nil
 }

src/cli/config_validation_test.go

@@ -22,22 +22,25 @@ import (
 func TestValidateNetworkingConfig(t *testing.T) {
 	t.Run("LocalAddr and LocalInterface both specified", func(t *testing.T) {
 		gc := &CLIConf{
-			LocalAddrString:  "1.1.1.1",
-			LocalIfaceString: "eth0",
+			LocalAddrString:   "1.1.1.1",
+			LocalIfaceString:  "eth0",
+			IPv4TransportOnly: true,
 		}
 		err := populateNetworkingConfig(gc)
 		require.NotNil(t, err, "Expected an error but got nil")
 	})
 	t.Run("Using invalid interface", func(t *testing.T) {
 		gc := &CLIConf{
-			LocalIfaceString: "invalid_interface",
+			LocalIfaceString:  "invalid_interface",
+			IPv4TransportOnly: true,
 		}
 		err := populateNetworkingConfig(gc)
 		require.NotNil(t, err, "Expected an error but got nil")
 	})
 	t.Run("Using nameserver with port", func(t *testing.T) {
 		gc := &CLIConf{
 			NameServersString: "127.0.0.1:53",
+			IPv4TransportOnly: true,
 		}
 		err := populateNetworkingConfig(gc)
 		require.Nil(t, err, "Expected no error but got %v", err)