Skip to content

Commit

Permalink
Add DoH and DoT support (#431)
Browse files Browse the repository at this point in the history
* added --https to cli

* working lookup to cloudflare.com, need to unpack tho

* response is parsed

* added zgrab's http module to get TLS info

* can print TLS stuff

* progress on proper int array handling

* made type cast work for all int/uint types

* comment and remove todo

* working dns-over-tls

* tls working

* go mod tidy

* fix compile errors

* compiles, doh not working

* doh and dot working w/o server cert validation

* dot working and printing out server handshake details

* avoid changing nameserver domain directly in doh lookup

* added integration test for tls connection re-creation

* 2 new integration tests testing doh and dot with name-server-mode, passing

* better default resolver handling for doh and dot

* added 2 new integration tests for doh/tls and fixed a bug

* removed unnecessary test output file

* revert unintended changes to multiple.ini file

* lint

* revert unnecessary change to multiple.ini

* use constants for doh default resolvers

* self PR review

* Preserve TCP/TLS/HTTPS Connections (#445)

* working input demultiplexor with tls

* handled tcp conns

* handle HTTPS de-multiplexing

* lint

* improved error msg if user only supplies IPv4 addresses and we fail config validation

* added AXFR edge case handling

* added comments

* if TCP connection is closed, re-open it

* don't loop in retrying tcp connection

* spelling

* close TCP conns in Close()

* trying multiple de-multiplexors

* Revert "trying multiple de-multiplexors"

This reverts commit 2cb7877.

* TEST - check how long non-network activity takes

* TEST - :(

* removed testing line

* trying giving the pool channels a capacity

* implement work-balancing scheme

* added small wait before going to global queue

* fix errors if destination closes the TCP connection

* lint

* refactor - coalesce language around worker channels

* removed the shouldRetryIfConnClosed bool, didn't add anything

* cleanup

* add ns stickiness but remove fancy channel stuff

* force to re-create TCP conn if it's closed by receiver

* remove log msg for debug

* added ServerCert verification

* lint

* better error handling around TLS failures and using correct default google server name
  • Loading branch information
phillip-stephens authored Sep 16, 2024
1 parent e9a0075 commit 1a8510d
Show file tree
Hide file tree
Showing 16 changed files with 1,555 additions and 236 deletions.
18 changes: 16 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -11,24 +11,38 @@ require (
github.com/stretchr/testify v1.9.0
github.com/zmap/dns v1.1.59-zdns-4
github.com/zmap/go-iptree v0.0.0-20210731043055-d4e632617837
github.com/zmap/zcrypto v0.0.0-20240803002437-3a861682ac77
github.com/zmap/zflags v1.4.0-beta.1.0.20200204220219-9d95409821b6
github.com/zmap/zgrab2 v0.1.8
gotest.tools/v3 v3.5.1
)

require (
github.com/asergeyev/nradix v0.0.0-20220715161825-e451993e425c // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_golang v1.14.0 // indirect
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.42.0 // indirect
github.com/prometheus/procfs v0.9.0 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/weppos/publicsuffix-go v0.40.2 // indirect
github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect
golang.org/x/crypto v0.25.0 // indirect
golang.org/x/mod v0.18.0 // indirect
golang.org/x/net v0.26.0 // indirect
golang.org/x/net v0.27.0 // indirect
golang.org/x/sync v0.7.0 // indirect
golang.org/x/sys v0.22.0 // indirect
golang.org/x/term v0.22.0 // indirect
golang.org/x/text v0.16.0 // indirect
golang.org/x/tools v0.22.0 // indirect
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
google.golang.org/protobuf v1.33.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
602 changes: 599 additions & 3 deletions go.sum

Large diffs are not rendered by default.

10 changes: 6 additions & 4 deletions src/cli/cli.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,8 @@ import (
log "github.com/sirupsen/logrus"
"github.com/zmap/dns"
flags "github.com/zmap/zflags"
)

const (
zdnsCLIVersion = "1.1.0"
"github.com/zmap/zdns/src/zdns"
)

var parser *flags.Parser
Expand Down Expand Up @@ -72,13 +70,17 @@ type QueryOptions struct {
type NetworkOptions struct {
IPv4TransportOnly bool `long:"4" description:"utilize IPv4 query transport only, incompatible with --6"`
IPv6TransportOnly bool `long:"6" description:"utilize IPv6 query transport only, incompatible with --4"`
DNSOverHTTPS bool `long:"https" description:"Use DNS over HTTPS for lookups, mutually exclusive with --udp-only, --iterative, and --tls"`
LocalAddrString string `long:"local-addr" description:"comma-delimited list of local addresses to use, serve as the source IP for outbound queries"`
LocalIfaceString string `long:"local-interface" description:"local interface to use"`
DisableRecycleSockets bool `long:"no-recycle-sockets" description:"do not create long-lived unbound UDP socket for each thread at launch and reuse for all (UDP) queries"`
PreferIPv4Iteration bool `long:"prefer-ipv4-iteration" description:"Prefer IPv4/A record lookups during iterative resolution. Ignored unless used with both IPv4 and IPv6 query transport"`
PreferIPv6Iteration bool `long:"prefer-ipv6-iteration" description:"Prefer IPv6/AAAA record lookups during iterative resolution. Ignored unless used with both IPv4 and IPv6 query transport"`
RootCAsFile string `long:"root-cas-file" description:"Path to a file containing PEM-encoded root CAs to use for verifying server certificates, required for --verify-server-cert"`
TCPOnly bool `long:"tcp-only" description:"Only perform lookups over TCP"`
DNSOverTLS bool `long:"tls" description:"Use DNS over TLS for lookups, mutually exclusive with --udp-only, --iterative, and --https"`
UDPOnly bool `long:"udp-only" description:"Only perform lookups over UDP"`
VerifyServerCert bool `long:"verify-server-cert" description:"Verify the server's certificate when using DNS over TLS or DNS over HTTPS"`
}

// InputOutputOptions options for controlling the input and output behavior of zdns. Applicable to all modules.
Expand Down Expand Up @@ -193,7 +195,7 @@ func parseArgs() {
parser.Options = flags.Default ^ flags.PrintErrors // we'll print errors in the 2nd invocation, otherwise we get the error printed twice
_, _, _, _ = parser.ParseCommandLine(os.Args[1:])
if GC.Version {
fmt.Printf("zdns version %s", zdnsCLIVersion)
fmt.Printf("zdns version %s", zdns.ZDNSVersion)
fmt.Println()
os.Exit(0)
}
Expand Down
20 changes: 20 additions & 0 deletions src/cli/config_validation.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,26 @@ func populateNetworkingConfig(gc *CLIConf) error {
return errors.New("--local-addr and --local-interface cannot both be specified")
}

if gc.DNSOverHTTPS && gc.IterativeResolution {
return errors.New("--https and --iterative cannot both be specified")
}

if gc.DNSOverTLS && gc.IterativeResolution {
return errors.New("--tls and --iterative cannot both be specified")
}

if gc.UDPOnly && gc.DNSOverHTTPS {
return errors.New("--udp-only and --https cannot both be specified")
}

if gc.UDPOnly && gc.DNSOverTLS {
return errors.New("--udp-only and --tls cannot both be specified")
}

if gc.DNSOverHTTPS && gc.DNSOverTLS {
return errors.New("--https and --tls cannot both be specified")
}

if err := parseNameServers(gc); err != nil {
return errors.Wrap(err, "name servers could not be parsed")
}
Expand Down
184 changes: 184 additions & 0 deletions src/cli/util.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,184 @@
/*
* ZDNS Copyright 2024 Regents of the University of Michigan
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy
* of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License.
*/

package cli

import (
"encoding/json"

"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)

// replaceIntSliceInterface replaces all slices of ints/uints with a JSON byte slice in the input interface
// this is needed because if you marshal a slice of interface{}'s, where the interface{} objects are ints, it'll
// get outputted as a list of numbers instead of a base64 encoded byte slice. This function recursively traverses
// the input interface and replaces all slices of ints/uints with a JSON byte slice, or leaves the input interface
// unchanged if it doesn't contain any slices of ints/uints
func replaceIntSliceInterface(data interface{}) interface{} {
// special case
jsonData, err := marshalIntSlice(data)
if err != nil {
log.Errorf("error marshalling data in int slice: %v", err)
return data
} else if jsonData != nil {
return jsonData
}

// map recursive case
if castedData, ok := data.(map[string]interface{}); ok {
for k, v := range castedData {
castedData[k] = replaceIntSliceInterface(v)
}
return castedData
}
// slice recursive case
if castedData, ok := data.([]interface{}); ok {
for i, v := range castedData {
castedData[i] = replaceIntSliceInterface(v)
}
return castedData
}
// general case
return data
}

// marshalIntSlice marshals a slice of ints, uints, or interfaces containing ints or uints into a JSON byte slice
// If the input is not a slice of ints, uints, or interfaces containing ints or uints, it returns nil, nil
func marshalIntSlice(v interface{}) ([]byte, error) {
switch v := v.(type) {
case []int:
return json.Marshal(v)
case []int8:
return json.Marshal(v)
case []int16:
return json.Marshal(v)
case []int32:
return json.Marshal(v)
case []int64:
return json.Marshal(v)
case []uint:
return json.Marshal(v)
case []uint8:
return json.Marshal(v)
case []uint16:
return json.Marshal(v)
case []uint32:
return json.Marshal(v)
case []uint64:
return json.Marshal(v)
case []interface{}:
var ok bool
if len(v) > 0 {
// Check the type of the first element
switch v[0].(type) {
case int:
converted := make([]int, len(v))
for i, val := range v {
converted[i], ok = val.(int)
if !ok {
return nil, errors.New("failed to convert interface to int")
}
}
return json.Marshal(converted)
case int8:
converted := make([]int8, len(v))
for i, val := range v {
converted[i], ok = val.(int8)
if !ok {
return nil, errors.New("failed to convert interface to int8")
}
}
return json.Marshal(converted)
case int16:
converted := make([]int16, len(v))
for i, val := range v {
converted[i], ok = val.(int16)
if !ok {
return nil, errors.New("failed to convert interface to int16")
}
}
return json.Marshal(converted)
case int32:
converted := make([]int32, len(v))
for i, val := range v {
converted[i], ok = val.(int32)
if !ok {
return nil, errors.New("failed to convert interface to int32")
}
}
return json.Marshal(converted)
case int64:
converted := make([]int64, len(v))
for i, val := range v {
converted[i], ok = val.(int64)
if !ok {
return nil, errors.New("failed to convert interface to int64")
}
}
return json.Marshal(converted)
case uint:
converted := make([]uint, len(v))
for i, val := range v {
converted[i], ok = val.(uint)
if !ok {
return nil, errors.New("failed to convert interface to uint")
}
}
return json.Marshal(converted)
case uint8:
converted := make([]uint8, len(v))
for i, val := range v {
converted[i], ok = val.(uint8)
if !ok {
return nil, errors.New("failed to convert interface to uint8")
}
}
return json.Marshal(converted)
case uint16:
converted := make([]uint16, len(v))
for i, val := range v {
converted[i], ok = val.(uint16)
if !ok {
return nil, errors.New("failed to convert interface to uint16")
}
}
return json.Marshal(converted)
case uint32:
converted := make([]uint32, len(v))
for i, val := range v {
converted[i], ok = val.(uint32)
if !ok {
return nil, errors.New("failed to convert interface to uint32")
}
}
return json.Marshal(converted)
case uint64:
converted := make([]uint64, len(v))
for i, val := range v {
converted[i], ok = val.(uint64)
if !ok {
return nil, errors.New("failed to convert interface to uint64")
}
}
return json.Marshal(converted)
default:
return nil, nil
}
}
return nil, nil
default:
return nil, nil
}
}
Loading

0 comments on commit 1a8510d

Please sign in to comment.