Add DoH and DoT support

go.mod

@@ -11,24 +11,38 @@ require (
 	github.com/stretchr/testify v1.9.0
 	github.com/zmap/dns v1.1.59-zdns-4
 	github.com/zmap/go-iptree v0.0.0-20210731043055-d4e632617837
+	github.com/zmap/zcrypto v0.0.0-20240803002437-3a861682ac77
 	github.com/zmap/zflags v1.4.0-beta.1.0.20200204220219-9d95409821b6
+	github.com/zmap/zgrab2 v0.1.8
 	gotest.tools/v3 v3.5.1
 )
 
 require (
 	github.com/asergeyev/nradix v0.0.0-20220715161825-e451993e425c // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/prometheus/client_golang v1.14.0 // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.42.0 // indirect
+	github.com/prometheus/procfs v0.9.0 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/weppos/publicsuffix-go v0.40.2 // indirect
+	github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect
+	golang.org/x/crypto v0.25.0 // indirect
 	golang.org/x/mod v0.18.0 // indirect
-	golang.org/x/net v0.26.0 // indirect
+	golang.org/x/net v0.27.0 // indirect
 	golang.org/x/sync v0.7.0 // indirect
 	golang.org/x/sys v0.22.0 // indirect
 	golang.org/x/term v0.22.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
 	golang.org/x/tools v0.22.0 // indirect
-	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

src/cli/cli.go

@@ -25,10 +25,8 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/zmap/dns"
 	flags "github.com/zmap/zflags"
-)
 
-const (
-	zdnsCLIVersion = "1.1.0"
+	"github.com/zmap/zdns/src/zdns"
 )
 
 var parser *flags.Parser
@@ -72,13 +70,17 @@ type QueryOptions struct {
 type NetworkOptions struct {
 	IPv4TransportOnly     bool   `long:"4" description:"utilize IPv4 query transport only, incompatible with --6"`
 	IPv6TransportOnly     bool   `long:"6" description:"utilize IPv6 query transport only, incompatible with --4"`
+	DNSOverHTTPS          bool   `long:"https" description:"Use DNS over HTTPS for lookups, mutually exclusive with --udp-only, --iterative, and --tls"`
 	LocalAddrString       string `long:"local-addr" description:"comma-delimited list of local addresses to use, serve as the source IP for outbound queries"`
 	LocalIfaceString      string `long:"local-interface" description:"local interface to use"`
 	DisableRecycleSockets bool   `long:"no-recycle-sockets" description:"do not create long-lived unbound UDP socket for each thread at launch and reuse for all (UDP) queries"`
 	PreferIPv4Iteration   bool   `long:"prefer-ipv4-iteration" description:"Prefer IPv4/A record lookups during iterative resolution. Ignored unless used with both IPv4 and IPv6 query transport"`
 	PreferIPv6Iteration   bool   `long:"prefer-ipv6-iteration" description:"Prefer IPv6/AAAA record lookups during iterative resolution. Ignored unless used with both IPv4 and IPv6 query transport"`
+	RootCAsFile           string `long:"root-cas-file" description:"Path to a file containing PEM-encoded root CAs to use for verifying server certificates, required for --verify-server-cert"`
 	TCPOnly               bool   `long:"tcp-only" description:"Only perform lookups over TCP"`
+	DNSOverTLS            bool   `long:"tls" description:"Use DNS over TLS for lookups, mutually exclusive with --udp-only, --iterative, and --https"`
 	UDPOnly               bool   `long:"udp-only" description:"Only perform lookups over UDP"`
+	VerifyServerCert      bool   `long:"verify-server-cert" description:"Verify the server's certificate when using DNS over TLS or DNS over HTTPS"`
 }
 
 // InputOutputOptions options for controlling the input and output behavior of zdns. Applicable to all modules.
@@ -193,7 +195,7 @@ func parseArgs() {
 	parser.Options = flags.Default ^ flags.PrintErrors // we'll print errors in the 2nd invocation, otherwise we get the error printed twice
 	_, _, _, _ = parser.ParseCommandLine(os.Args[1:])
 	if GC.Version {
-		fmt.Printf("zdns version %s", zdnsCLIVersion)
+		fmt.Printf("zdns version %s", zdns.ZDNSVersion)
 		fmt.Println()
 		os.Exit(0)
 	}

src/cli/config_validation.go

@@ -31,6 +31,26 @@ func populateNetworkingConfig(gc *CLIConf) error {
 		return errors.New("--local-addr and --local-interface cannot both be specified")
 	}
 
+	if gc.DNSOverHTTPS && gc.IterativeResolution {
+		return errors.New("--https and --iterative cannot both be specified")
+	}
+
+	if gc.DNSOverTLS && gc.IterativeResolution {
+		return errors.New("--tls and --iterative cannot both be specified")
+	}
+
+	if gc.UDPOnly && gc.DNSOverHTTPS {
+		return errors.New("--udp-only and --https cannot both be specified")
+	}
+
+	if gc.UDPOnly && gc.DNSOverTLS {
+		return errors.New("--udp-only and --tls cannot both be specified")
+	}
+
+	if gc.DNSOverHTTPS && gc.DNSOverTLS {
+		return errors.New("--https and --tls cannot both be specified")
+	}
+
 	if err := parseNameServers(gc); err != nil {
 		return errors.Wrap(err, "name servers could not be parsed")
 	}

src/cli/util.go

@@ -0,0 +1,184 @@
+/*
+ * ZDNS Copyright 2024 Regents of the University of Michigan
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package cli
+
+import (
+	"encoding/json"
+
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
+)
+
+// replaceIntSliceInterface replaces all slices of ints/uints with a JSON byte slice in the input interface
+// this is needed because if you marshal a slice of interface{}'s, where the interface{} objects are ints, it'll
+// get outputted as a list of numbers instead of a base64 encoded byte slice. This function recursively traverses
+// the input interface and replaces all slices of ints/uints with a JSON byte slice, or leaves the input interface
+// unchanged if it doesn't contain any slices of ints/uints
+func replaceIntSliceInterface(data interface{}) interface{} {
+	// special case
+	jsonData, err := marshalIntSlice(data)
+	if err != nil {
+		log.Errorf("error marshalling data in int slice: %v", err)
+		return data
+	} else if jsonData != nil {
+		return jsonData
+	}
+
+	// map recursive case
+	if castedData, ok := data.(map[string]interface{}); ok {
+		for k, v := range castedData {
+			castedData[k] = replaceIntSliceInterface(v)
+		}
+		return castedData
+	}
+	// slice recursive case
+	if castedData, ok := data.([]interface{}); ok {
+		for i, v := range castedData {
+			castedData[i] = replaceIntSliceInterface(v)
+		}
+		return castedData
+	}
+	// general case
+	return data
+}
+
+// marshalIntSlice marshals a slice of ints, uints, or interfaces containing ints or uints into a JSON byte slice
+// If the input is not a slice of ints, uints, or interfaces containing ints or uints, it returns nil, nil
+func marshalIntSlice(v interface{}) ([]byte, error) {
+	switch v := v.(type) {
+	case []int:
+		return json.Marshal(v)
+	case []int8:
+		return json.Marshal(v)
+	case []int16:
+		return json.Marshal(v)
+	case []int32:
+		return json.Marshal(v)
+	case []int64:
+		return json.Marshal(v)
+	case []uint:
+		return json.Marshal(v)
+	case []uint8:
+		return json.Marshal(v)
+	case []uint16:
+		return json.Marshal(v)
+	case []uint32:
+		return json.Marshal(v)
+	case []uint64:
+		return json.Marshal(v)
+	case []interface{}:
+		var ok bool
+		if len(v) > 0 {
+			// Check the type of the first element
+			switch v[0].(type) {
+			case int:
+				converted := make([]int, len(v))
+				for i, val := range v {
+					converted[i], ok = val.(int)
+					if !ok {
+						return nil, errors.New("failed to convert interface to int")
+					}
+				}
+				return json.Marshal(converted)
+			case int8:
+				converted := make([]int8, len(v))
+				for i, val := range v {
+					converted[i], ok = val.(int8)
+					if !ok {
+						return nil, errors.New("failed to convert interface to int8")
+					}
+				}
+				return json.Marshal(converted)
+			case int16:
+				converted := make([]int16, len(v))
+				for i, val := range v {
+					converted[i], ok = val.(int16)
+					if !ok {
+						return nil, errors.New("failed to convert interface to int16")
+					}
+				}
+				return json.Marshal(converted)
+			case int32:
+				converted := make([]int32, len(v))
+				for i, val := range v {
+					converted[i], ok = val.(int32)
+					if !ok {
+						return nil, errors.New("failed to convert interface to int32")
+					}
+				}
+				return json.Marshal(converted)
+			case int64:
+				converted := make([]int64, len(v))
+				for i, val := range v {
+					converted[i], ok = val.(int64)
+					if !ok {
+						return nil, errors.New("failed to convert interface to int64")
+					}
+				}
+				return json.Marshal(converted)
+			case uint:
+				converted := make([]uint, len(v))
+				for i, val := range v {
+					converted[i], ok = val.(uint)
+					if !ok {
+						return nil, errors.New("failed to convert interface to uint")
+					}
+				}
+				return json.Marshal(converted)
+			case uint8:
+				converted := make([]uint8, len(v))
+				for i, val := range v {
+					converted[i], ok = val.(uint8)
+					if !ok {
+						return nil, errors.New("failed to convert interface to uint8")
+					}
+				}
+				return json.Marshal(converted)
+			case uint16:
+				converted := make([]uint16, len(v))
+				for i, val := range v {
+					converted[i], ok = val.(uint16)
+					if !ok {
+						return nil, errors.New("failed to convert interface to uint16")
+					}
+				}
+				return json.Marshal(converted)
+			case uint32:
+				converted := make([]uint32, len(v))
+				for i, val := range v {
+					converted[i], ok = val.(uint32)
+					if !ok {
+						return nil, errors.New("failed to convert interface to uint32")
+					}
+				}
+				return json.Marshal(converted)
+			case uint64:
+				converted := make([]uint64, len(v))
+				for i, val := range v {
+					converted[i], ok = val.(uint64)
+					if !ok {
+						return nil, errors.New("failed to convert interface to uint64")
+					}
+				}
+				return json.Marshal(converted)
+			default:
+				return nil, nil
+			}
+		}
+		return nil, nil
+	default:
+		return nil, nil
+	}
+}