Add eTLD Handling in registered_domain Processor (#23046)

* Add eTLD handling in registered_domain_processor * Regenerate sysmon, zeek, and suricata golden files * Add changelog entry * Add empty string check
elastic · Dec 10, 2020 · e9d12e2 · e9d12e2
1 parent 65c9928
commit e9d12e2
Show file tree

Hide file tree

Showing 10 changed files with 351 additions and 166 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -755,6 +755,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Preserve AWS CloudTrail eventCategory in aws.cloudtrail.event_category. {issue}22776[22776] {pull}22805[22805]
 - Migrate microsoft/defender_atp to httpjson v2 config {pull}23017[23017]
 - Migrate microsoft/m365_defender to httpjson v2 config {pull}23018[23018]
+- Add top_level_domain enrichment for suricata/eve fileset. {pull}23046[23046]
+- Add top_level_domain enrichment for zeek/dns fileset. {pull}23046[23046]
 
 *Heartbeat*
 
@@ -913,6 +915,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add file.pe and process.pe fields to ProcessCreate & LoadImage events in Sysmon module. {issue}17335[17335] {pull}22217[22217]
 - Add additional event categorization for security and sysmon modules. {pull}22988[22988]
 - Add dns.question.subdomain fields for sysmon DNS events. {pull}22999[22999]
+- Add dns.question.top_level_domain fields for sysmon DNS events. {pull}23046[23046]
 
 *Elastic Log Driver*
 

diff --git a/libbeat/processors/registered_domain/config.go b/libbeat/processors/registered_domain/config.go
@@ -21,6 +21,7 @@ type config struct {
 	Field                string `config:"field"        validate:"required"`
 	TargetField          string `config:"target_field" validate:"required"`
 	TargetSubdomainField string `config:"target_subdomain_field"`
+	TargetETLDField      string `config:"target_etld_field"`
 	IgnoreMissing        bool   `config:"ignore_missing"`
 	IgnoreFailure        bool   `config:"ignore_failure"`
 	ID                   string `config:"id"`

diff --git a/libbeat/processors/registered_domain/docs/registered_domain.asciidoc b/libbeat/processors/registered_domain/docs/registered_domain.asciidoc
@@ -20,6 +20,7 @@ processors:
   - registered_domain:
       field: dns.question.name
       target_field: dns.question.registered_domain
+      target_etld_field: dns.question.top_level_domain
       target_subdomain_field: dns.question.sudomain
       ignore_missing: true
       ignore_failure: true
@@ -33,6 +34,7 @@ The `registered_domain` processor has the following configuration settings:
 | Name                     | Required | Default    | Description                                                      |
 | `field`                  | yes      |            | Source field containing a fully qualified domain name (FQDN).    |
 | `target_field`           | yes      |            | Target field for the registered domain value.                    |
+| `target_etld_field`      | no       |            | Target field for the effective top-level domain value.          |
 | `target_subdomain_field` | no       |            | Target subdomain field for the subdomain value.                  |
 | `ignore_missing`         | no       | false      | Ignore errors when the source field is missing.                  |
 | `ignore_failure`         | no       | false      | Ignore all errors produced by the processor.                     |

diff --git a/libbeat/processors/registered_domain/registered_domain.go b/libbeat/processors/registered_domain/registered_domain.go
@@ -106,6 +106,15 @@ func (p *processor) Run(event *beat.Event) (*beat.Event, error) {
 		return event, errors.Wrapf(err, "failed to write registered domain to target field [%v]", p.TargetField)
 	}
 
+	if p.TargetETLDField != "" {
+		tld, _ := publicsuffix.PublicSuffix(domain)
+		if tld != "" {
+			if _, err = event.PutValue(p.TargetETLDField, tld); err != nil && !p.IgnoreFailure {
+				return event, errors.Wrapf(err, "failed to write effective top-level domain to target field [%v]", p.TargetETLDField)
+			}
+		}
+	}
+
 	if p.TargetSubdomainField != "" {
 		subdomain := strings.TrimSuffix(strings.TrimSuffix(domain, rd), ".")
 		if subdomain != "" {

diff --git a/libbeat/processors/registered_domain/registered_domain_test.go b/libbeat/processors/registered_domain/registered_domain_test.go
@@ -32,24 +32,26 @@ func TestProcessorRun(t *testing.T) {
 		Domain           string
 		RegisteredDomain string
 		Subdomain        string
+		ETLD             string
 	}{
-		{false, "www.google.com", "google.com", "www"},
-		{false, "www.google.co.uk", "google.co.uk", "www"},
-		{false, "www.mail.google.co.uk", "google.co.uk", "www.mail"},
-		{false, "google.com", "google.com", ""},
-		{false, "www.ak.local", "ak.local", "www"},
-		{false, "www.navy.mil", "navy.mil", "www"},
+		{false, "www.google.com", "google.com", "www", "com"},
+		{false, "www.google.co.uk", "google.co.uk", "www", "co.uk"},
+		{false, "www.mail.google.co.uk", "google.co.uk", "www.mail", "co.uk"},
+		{false, "google.com", "google.com", "", "com"},
+		{false, "www.ak.local", "ak.local", "www", "local"},
+		{false, "www.navy.mil", "navy.mil", "www", "mil"},
 
-		{true, "com", "", ""},
-		{true, ".", ".", ""},
-		{true, "", "", ""},
-		{true, "localhost", "", ""},
+		{true, "com", "", "", ""},
+		{true, ".", ".", "", ""},
+		{true, "", "", "", ""},
+		{true, "localhost", "", "", ""},
 	}
 
 	c := defaultConfig()
 	c.Field = "domain"
 	c.TargetField = "registered_domain"
 	c.TargetSubdomainField = "subdomain"
+	c.TargetETLDField = "etld"
 	p, err := newRegisteredDomain(c)
 	if err != nil {
 		t.Fatal(err)
@@ -75,9 +77,20 @@ func TestProcessorRun(t *testing.T) {
 		rd, _ := evt.GetValue("registered_domain")
 		assert.Equal(t, tc.RegisteredDomain, rd)
 
-		if tc.Subdomain != "" {
+		if tc.Subdomain == "" {
+			_, err := evt.GetValue("subdomain")
+			assert.NotNil(t, err)
+		} else {
 			subdomain, _ := evt.GetValue("subdomain")
 			assert.Equal(t, tc.Subdomain, subdomain)
 		}
+
+		if tc.ETLD == "" {
+			_, err := evt.GetValue("etld")
+			assert.NotNil(t, err)
+		} else {
+			etld, _ := evt.GetValue("etld")
+			assert.Equal(t, tc.ETLD, etld)
+		}
 	}
 }
diff --git a/x-pack/filebeat/module/suricata/eve/config/eve.yml b/x-pack/filebeat/module/suricata/eve/config/eve.yml
@@ -53,6 +53,7 @@ processors:
       field: suricata.eve.dns.rrname
       target_field: dns.question.registered_domain
       target_subdomain_field: dns.question.subdomain
+      target_etld_field: dns.question.top_level_domain
   - add_fields:
       target: ''
       fields:

diff --git a/x-pack/filebeat/module/zeek/dns/config/dns.yml b/x-pack/filebeat/module/zeek/dns/config/dns.yml
@@ -20,6 +20,7 @@ processors:
       field: zeek.dns.query
       target_field: dns.question.registered_domain
       target_subdomain_field: dns.question.subdomain
+      target_etld_field: dns.question.top_level_domain
   - script:
       lang: javascript
       id: zeek_dns_flags

diff --git a/x-pack/winlogbeat/module/sysmon/config/winlogbeat-sysmon.js b/x-pack/winlogbeat/module/sysmon/config/winlogbeat-sysmon.js
@@ -1564,6 +1564,7 @@ var sysmon = (function () {
             field: "dns.question.name",
             target_field: "dns.question.registered_domain",
             target_subdomain_field: "dns.question.subdomain",
+            target_etld_field: "dns.question.top_level_domain",
         })
         .Add(setRuleName)
         .Add(translateDnsQueryStatus)