DNS over TLS support for DNS processor (#19321) (#20090)

* DNS over TLS DoT support #16663

    * added changelog #19321

    * Update dns.asciidoc

    * added testsuite for #19321

    * Fix CHANGELOG entries and lint fixes

    * Apply suggestions from code review

    Co-authored-by: Marc Guasch <marc.guasch@elastic.co>
    Co-authored-by: Andrew Kroh <andrew.kroh@elastic.co>
    (cherry picked from commit 89bfb6c)

Co-authored-by: Andrew Kroh <andrew.kroh@elastic.co>

CHANGELOG.next.asciidoc

@@ -436,6 +436,7 @@ field. You can revert this change by configuring tags for the module and omittin
 - Add the `overwrite_keys` configuration option to the dissect processor. {pull}19464[19464]
 - Add support to trim captured values in the dissect processor. {pull}19464[19464]
 - Added the `max_cached_sessions` option to the script processor. {pull}19562[19562]
+- Add support for DNS over TLS for the `dns` processor. {pull}19321[19321]
 - Set index.max_docvalue_fields_search in index template to increase value to 200 fields. {issue}20215[20215]
 - Add leader election for Kubernetes autodiscover. {pull}20281[20281]
 - Add capability of enriching process metadata with contianer id also for non-privileged containers in `add_process_metadata` processor. {pull}19767[19767]

libbeat/processors/dns/config.go

@@ -36,6 +36,7 @@ type Config struct {
 	Action       FieldAction   `config:"action"`                   // Append or replace (defaults to append) when target exists.
 	TagOnFailure []string      `config:"tag_on_failure"`           // Tags to append when a failure occurs.
 	Fields       common.MapStr `config:"fields"`                   // Mapping of source fields to target fields.
+	Transport    string        `config:"transport"`                // Can be tls or udp.
 	reverseFlat  map[string]string
 }
 
@@ -117,6 +118,14 @@ func (c *Config) Validate() error {
 		c.reverseFlat[k] = target
 	}
 
+	c.Transport = strings.ToLower(c.Transport)
+	switch c.Transport {
+	case "tls":
+	case "udp":
+	default:
+		return errors.Errorf("invalid transport method type '%v' specified in "+
+			"config (valid value is: tls or udp)", c.Transport)
+	}
 	return nil
 }
 
@@ -155,5 +164,6 @@ var defaultConfig = Config{
 			MaxCapacity:     10000,
 		},
 	},
-	Timeout: 500 * time.Millisecond,
+	Transport: "udp",
+	Timeout:   500 * time.Millisecond,
 }

libbeat/processors/dns/dns.go

@@ -65,7 +65,7 @@ func New(cfg *common.Config) (processors.Processor, error) {
 	)
 
 	log.Debugf("DNS processor config: %+v", c)
-	resolver, err := NewMiekgResolver(metrics, c.Timeout, c.Nameservers...)
+	resolver, err := NewMiekgResolver(metrics, c.Timeout, c.Transport, c.Nameservers...)
 	if err != nil {
 		return nil, err
 	}

libbeat/processors/dns/docs/dns.asciidoc

@@ -45,6 +45,7 @@ processors:
 - dns:
     type: reverse
     action: append
+    transport: tls
     fields:
       server.ip: server.hostname
       client.ip: client.hostname
@@ -104,3 +105,6 @@ for each DNS request so if you have 2 nameservers then the total timeout will be
 `tag_on_failure`:: A list of tags to add to the event when any lookup fails. The
 tags are only added once even if multiple lookups fail. By default no tags are
 added upon failure.
+
+`transport`:: The type of transport connection that should be used can either be
+`tls` (DNS over TLS) or `udp`. Defaults to `udp`.

libbeat/processors/dns/resolver.go

@@ -64,7 +64,7 @@ type nameserverStats struct {
 
 // NewMiekgResolver returns a new MiekgResolver. It returns an error if no
 // nameserver are given and none can be read from /etc/resolv.conf.
-func NewMiekgResolver(reg *monitoring.Registry, timeout time.Duration, servers ...string) (*MiekgResolver, error) {
+func NewMiekgResolver(reg *monitoring.Registry, timeout time.Duration, transport string, servers ...string) (*MiekgResolver, error) {
 	// Use /etc/resolv.conf if no nameservers are given. (Won't work for Windows).
 	if len(servers) == 0 {
 		config, err := dns.ClientConfigFromFile(etcResolvConf)
@@ -77,7 +77,14 @@ func NewMiekgResolver(reg *monitoring.Registry, timeout time.Duration, servers .
 	// Add port if one was not specified.
 	for i, s := range servers {
 		if _, _, err := net.SplitHostPort(s); err != nil {
-			withPort := s + ":53"
+			var withPort string
+			switch transport {
+			case "tls":
+				withPort = s + ":853"
+			default:
+				withPort = s + ":53"
+			}
+
 			if _, _, retryErr := net.SplitHostPort(withPort); retryErr == nil {
 				servers[i] = withPort
 				continue
@@ -90,9 +97,17 @@ func NewMiekgResolver(reg *monitoring.Registry, timeout time.Duration, servers .
 		timeout = defaultConfig.Timeout
 	}
 
+	var clientTransferType string
+	switch transport {
+	case "tls":
+		clientTransferType = "tcp-tls"
+	default:
+		clientTransferType = "udp"
+	}
+
 	return &MiekgResolver{
 		client: &dns.Client{
-			Net:     "udp",
+			Net:     clientTransferType,
 			Timeout: timeout,
 		},
 		servers:  servers,

libbeat/processors/dns/resolver_test.go

@@ -18,6 +18,7 @@
 package dns
 
 import (
+	"crypto/tls"
 	"net"
 	"strings"
 	"testing"
@@ -38,7 +39,7 @@ func TestMiekgResolverLookupPTR(t *testing.T) {
 	defer stop()
 
 	reg := monitoring.NewRegistry()
-	res, err := NewMiekgResolver(reg.NewRegistry(logName), 0, addr)
+	res, err := NewMiekgResolver(reg.NewRegistry(logName), 0, "udp", addr)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -68,8 +69,61 @@ func TestMiekgResolverLookupPTR(t *testing.T) {
 	assert.Equal(t, 12, metricCount)
 }
 
+func TestMiekgResolverLookupPTRTLS(t *testing.T) {
+	//Build Cert
+	cert, err := tls.X509KeyPair(CertPEMBlock, KeyPEMBlock)
+	if err != nil {
+		t.Fatalf("unable to build certificate: %v", err)
+	}
+	config := tls.Config{
+		Certificates: []tls.Certificate{cert},
+	}
+	// serve TLS with cert
+	stop, addr, err := ServeDNSTLS(FakeDNSHandler, &config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer stop()
+
+	reg := monitoring.NewRegistry()
+
+	res, err := NewMiekgResolver(reg.NewRegistry(logName), 0, "tls", addr)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// we use a self signed certificate for localhost
+	// we have to pass InsecureSSL to the DNS resolver
+	res.client.TLSConfig = &tls.Config{
+		InsecureSkipVerify: true,
+	}
+	// Success
+	ptr, err := res.LookupPTR("8.8.8.8")
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.EqualValues(t, "google-public-dns-a.google.com", ptr.Host)
+	assert.EqualValues(t, 19273, ptr.TTL)
+
+	// NXDOMAIN
+	_, err = res.LookupPTR("1.1.1.1")
+	if assert.Error(t, err) {
+		assert.Contains(t, err.Error(), "NXDOMAIN")
+	}
+
+	// Validate that our metrics exist.
+	var metricCount int
+	reg.Do(monitoring.Full, func(name string, v interface{}) {
+		if strings.Contains(name, "processor.dns") {
+			metricCount++
+		}
+		t.Logf("%v: %+v", name, v)
+	})
+	assert.Equal(t, 12, metricCount)
+}
+
 func ServeDNS(h dns.HandlerFunc) (cancel func() error, addr string, err error) {
 	// Setup listener on ephemeral port.
+
 	a, err := net.ResolveUDPAddr("udp4", "localhost:0")
 	if err != nil {
 		return nil, "", err
@@ -86,6 +140,20 @@ func ServeDNS(h dns.HandlerFunc) (cancel func() error, addr string, err error) {
 	return s.Shutdown, s.PacketConn.LocalAddr().String(), err
 }
 
+func ServeDNSTLS(h dns.HandlerFunc, config *tls.Config) (cancel func() error, addr string, err error) {
+	// Setup listener on ephemeral port.
+	l, err := tls.Listen("tcp", "localhost:0", config)
+	if err != nil {
+		return nil, "", err
+	}
+
+	var s dns.Server
+	s.Handler = h
+	s.Listener = l
+	go s.ActivateAndServe()
+	return s.Shutdown, l.Addr().String(), err
+}
+
 func FakeDNSHandler(w dns.ResponseWriter, msg *dns.Msg) {
 	m := new(dns.Msg)
 	m.SetReply(msg)
@@ -98,3 +166,55 @@ func FakeDNSHandler(w dns.ResponseWriter, msg *dns.Msg) {
 	}
 	w.WriteMsg(m)
 }
+
+var (
+	KeyPEMBlock = []byte(`-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEA2g2zpEtWaIUx5o6MEnWnGsf0Ba1SDc3AwgOmxeNIPBJYVCrk
+sWe8Qt/5nymReVFcum76995ncr/zT+e4e8l+hXuGzTKZJpOj27Igb0/wa3j2hIcu
+rnbzfwkJ+KMag2UUKdSo31ChMU+64bwziEXunF347Ot7dBLtw3PJKbabNCP+/oil
+iUv2TzxxYosN+AEg4gNKLa3DMpbUnD+9Igb9KmaVp1FVhZted/AP4vn7h6Urb4ER
+xMuvv3xqZvIKQ9/G1XAISYXk2feZ5yP+k1HF4ds7HJDwrP+Bv+EVyv38EKdmu9N3
+Oej8wKf3Acjln/ucbg1S3Dmkyg0x2388S4c35wIDAQABAoIBAB8MnGvknmU7siNW
+YPOv9R+HIWQ9jdWRWsVFp9W9y2diZVl20iHA17neErlrPd+8iiux6eKptKlOU+Mo
+58gYpP9023kUn2Iy275I2v1+sIldLB0q8qa9IWcRbm4NK5VSK1DZi0JhRNK0u7Ox
+DNV2v8dcSjnSPj4FA/402owqCGegBQuheYE0LDEMiNAm6hZmQ5Npf0mTfJA/OuM4
+ONSR7lNncrR0pOZ3f3WWH+021eoZCgu2A64yfX5FFI7y5jvRn8KigXEDfXcdyFKO
+725Slq4V2E2NmrMyRKNBLUSUC2hcy0tQsfo3+yANxA6PBNQ0EVqkF4uGn1IzNWOz
+gDSyfSECgYEA2jgTpv9v0SrURdY3lOOjYZNCoJ9ZhUTxOsQQZLUJ+1/bQQ4Y0ONK
+cnC/Ve76C/k+otbILAaRnOxGw5Apq25yPNoxjFFzP7tbN85IB+4db637qZNK2gfX
+oEJd6wat4Urs8NbUKCE+XkbdENOIdXUiQxp9U6jXxprd5Ii4jICwRvsCgYEA/85J
+1to++Td64gKfWDv4FUo5ZqVn70JdM/Knf5Pd37z/sjNowxhDz7AhismRditX02lt
+T2g/raIW9Z/SpxI44VHCRJGPOvBvaMgCNGOH0FBHatFsfKwKzpMwapTfobqj3ZYa
+DDDc8r9WQM8IDcLM6B7aOV46LWMEhMRSfDa9bwUCgYEAokbRVn7eSE3xTX3gF3ix
+Jv67rXbSu6hpO6pSBpIaujSud9Jj4fMkibYOk3kDuaPAUJgog5Te9DNA7G1oj3Oy
+wE4CSrbHXb2WOAnOxxbsDQD1BUXjhAAQ+bxg20Y8SC3Pxcn8O1t9Zd6MxtaHw9E3
+iW9Jg80rqSXBnRGPK+0HKcECgYBsRYk1WjzLSTNG1CtTslZH1JnFG3+JYoKGiU9i
+DVkc6Sck6uONqAiTsI4R600ZQjEzN21f7dT+Dhw/rH0B4BGZNPzP/vgrzzaol/du
+6y3B+yivSqLrhfoxA1W71vVsw8217WFrBYePa3L7jWVwRaJrIRvmqj5flYiFFX+A
+Ob8mbQKBgAHhlnVzoKCq4mZ7Glpc0K6L57btVZNn0TEGyVli1ECvgC3zRm1rEofG
+LatVl7h6ud25ZJYnP7DelGxHsZnDXNirLFlSB0CL4F6I5xNoBvCoH0Q8ckDSh4C7
+tlAyD5m9gwvgdkNFWq6/lcUPxGksTtTk8dGnhJz8pGlZvp6+dZCM
+-----END RSA PRIVATE KEY-----`)
+
+	CertPEMBlock = []byte(`-----BEGIN CERTIFICATE-----
+MIIDaTCCAlGgAwIBAgIQGqg47wLgbjwwrZASuakmwjANBgkqhkiG9w0BAQsFADAy
+MRQwEgYDVQQKEwtMb2cgQ291cmllcjEaMBgGA1UEAxMRYmVhdHMuZWxhc3RpYy5j
+b20wHhcNMjAwNjIzMDY0NDEwWhcNMjEwNjIzMDY0NDEwWjAyMRQwEgYDVQQKEwtM
+b2cgQ291cmllcjEaMBgGA1UEAxMRYmVhdHMuZWxhc3RpYy5jb20wggEiMA0GCSqG
+SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDaDbOkS1ZohTHmjowSdacax/QFrVINzcDC
+A6bF40g8ElhUKuSxZ7xC3/mfKZF5UVy6bvr33mdyv/NP57h7yX6Fe4bNMpkmk6Pb
+siBvT/BrePaEhy6udvN/CQn4oxqDZRQp1KjfUKExT7rhvDOIRe6cXfjs63t0Eu3D
+c8kptps0I/7+iKWJS/ZPPHFiiw34ASDiA0otrcMyltScP70iBv0qZpWnUVWFm153
+8A/i+fuHpStvgRHEy6+/fGpm8gpD38bVcAhJheTZ95nnI/6TUcXh2zsckPCs/4G/
+4RXK/fwQp2a703c56PzAp/cByOWf+5xuDVLcOaTKDTHbfzxLhzfnAgMBAAGjezB5
+MA4GA1UdDwEB/wQEAwICpDATBgNVHSUEDDAKBggrBgEFBQcDATAPBgNVHRMBAf8E
+BTADAQH/MEEGA1UdEQQ6MDiCATqCCWxvY2FsaG9zdIcQAAAAAAAAAAAAAAAAAAAA
+AIcEfwAAAYcQAAAAAAAAAAAAAAAAAAAAATANBgkqhkiG9w0BAQsFAAOCAQEAL6px
+cjflhqqewqa9cvhFNT6E7UDnA7Mf34GIQPQrORXyOnyE11mDp5sEMGaz8bDajHHc
+0JL8Q/5rDyRsSfe1pIyViAOxn+V/7qXfgowI3tkJbSaqHX7SlHF0dEiuGQ1coBMx
+RgW17XhPtV+fk/DiXtUEkgtB7/q0Kc9C9C2GJIbOtupZ/mnkdk/5YT4tfXywNnWC
+lLjT6T5+wZgRkcnr7lYNiTdS+GtN0YspPT+YD3ZTJCYD9KPcbA6k9XXXwmU8Ij6H
+waodyGzG03YJbY3l2zSt3lG3jv9Tj+Ic0kRyEzzxk8exyi6nWXue/6a884kgAjiL
+bXmdL6wkIJz1U+XtuQ==
+-----END CERTIFICATE-----`)
+)