Add eTLD Handling in registered_domain Processor

CHANGELOG.next.asciidoc

@@ -753,6 +753,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add `event.category` "configuration" to zoom module events. {pull}23010[23010]
 - Add `network.direction` to auditd/log fileset. {pull}23041[23041]
 - Preserve AWS CloudTrail eventCategory in aws.cloudtrail.event_category. {issue}22776[22776] {pull}22805[22805]
+- Add top_level_domain enrichment for suricata/eve fileset. {pull}23046[23046]
+- Add top_level_domain enrichment for zeek/dns fileset. {pull}23046[23046]
 
 *Heartbeat*
 
@@ -909,6 +911,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add file.pe and process.pe fields to ProcessCreate & LoadImage events in Sysmon module. {issue}17335[17335] {pull}22217[22217]
 - Add additional event categorization for security and sysmon modules. {pull}22988[22988]
 - Add dns.question.subdomain fields for sysmon DNS events. {pull}22999[22999]
+- Add dns.question.top_level_domain fields for sysmon DNS events. {pull}23046[23046]
 
 *Elastic Log Driver*
 

libbeat/processors/registered_domain/config.go

@@ -21,6 +21,7 @@ type config struct {
  Field string `config:"field" validate:"required"`
  TargetField string `config:"target_field" validate:"required"`
  TargetSubdomainField string `config:"target_subdomain_field"`
+ TargetETLDField string `config:"target_etld_field"`
  IgnoreMissing bool `config:"ignore_missing"`
  IgnoreFailure bool `config:"ignore_failure"`
  ID string `config:"id"`

libbeat/processors/registered_domain/docs/registered_domain.asciidoc

@@ -20,6 +20,7 @@ processors:
  - registered_domain:
  field: dns.question.name
  target_field: dns.question.registered_domain
+ target_etld_field: dns.question.top_level_domain
  target_subdomain_field: dns.question.sudomain
  ignore_missing: true
  ignore_failure: true
@@ -33,6 +34,7 @@ The `registered_domain` processor has the following configuration settings:
 | Name | Required | Default | Description |
 | `field` | yes | | Source field containing a fully qualified domain name (FQDN). |
 | `target_field` | yes | | Target field for the registered domain value. |
+| `target_etld_field` | no | | Target field for the effective top-level domain value. |
 | `target_subdomain_field` | no | | Target subdomain field for the subdomain value. |
 | `ignore_missing` | no | false | Ignore errors when the source field is missing. |
 | `ignore_failure` | no | false | Ignore all errors produced by the processor. |

libbeat/processors/registered_domain/registered_domain.go

@@ -106,6 +106,15 @@ func (p *processor) Run(event *beat.Event) (*beat.Event, error) {
  return event, errors.Wrapf(err, "failed to write registered domain to target field [%v]", p.TargetField)
  }
 
+ if p.TargetETLDField != "" {
+ tld, _ := publicsuffix.PublicSuffix(domain)
+ if tld != "" {
+ if _, err = event.PutValue(p.TargetETLDField, tld); err != nil && !p.IgnoreFailure {
+ return event, errors.Wrapf(err, "failed to write effective top-level domain to target field [%v]", p.TargetETLDField)
+ }
+ }
+ }
+
  if p.TargetSubdomainField != "" {
  subdomain := strings.TrimSuffix(strings.TrimSuffix(domain, rd), ".")
  if subdomain != "" {

libbeat/processors/registered_domain/registered_domain_test.go

@@ -32,24 +32,26 @@ func TestProcessorRun(t *testing.T) {
  Domain string
  RegisteredDomain string
  Subdomain string
+ ETLD string
  }{
- {false, "www.google.com", "google.com", "www"},
- {false, "www.google.co.uk", "google.co.uk", "www"},
- {false, "www.mail.google.co.uk", "google.co.uk", "www.mail"},
- {false, "google.com", "google.com", ""},
- {false, "www.ak.local", "ak.local", "www"},
- {false, "www.navy.mil", "navy.mil", "www"},
+ {false, "www.google.com", "google.com", "www", "com"},
+ {false, "www.google.co.uk", "google.co.uk", "www", "co.uk"},
+ {false, "www.mail.google.co.uk", "google.co.uk", "www.mail", "co.uk"},
+ {false, "google.com", "google.com", "", "com"},
+ {false, "www.ak.local", "ak.local", "www", "local"},
+ {false, "www.navy.mil", "navy.mil", "www", "mil"},
 
- {true, "com", "", ""},
- {true, ".", ".", ""},
- {true, "", "", ""},
- {true, "localhost", "", ""},
+ {true, "com", "", "", ""},
+ {true, ".", ".", "", ""},
+ {true, "", "", "", ""},
+ {true, "localhost", "", "", ""},
  }
 
  c := defaultConfig()
  c.Field = "domain"
  c.TargetField = "registered_domain"
  c.TargetSubdomainField = "subdomain"
+ c.TargetETLDField = "etld"
  p, err := newRegisteredDomain(c)
  if err != nil {
  t.Fatal(err)
@@ -75,9 +77,20 @@ func TestProcessorRun(t *testing.T) {
  rd, _ := evt.GetValue("registered_domain")
  assert.Equal(t, tc.RegisteredDomain, rd)
 
- if tc.Subdomain != "" {
+ if tc.Subdomain == "" {
+ _, err := evt.GetValue("subdomain")
+ assert.NotNil(t, err)
+ } else {
  subdomain, _ := evt.GetValue("subdomain")
  assert.Equal(t, tc.Subdomain, subdomain)
  }
+
+ if tc.ETLD == "" {
+ _, err := evt.GetValue("etld")
+ assert.NotNil(t, err)
+ } else {
+ etld, _ := evt.GetValue("etld")
+ assert.Equal(t, tc.ETLD, etld)
+ }
  }
 }

x-pack/filebeat/module/suricata/eve/config/eve.yml

@@ -53,6 +53,7 @@ processors:
  field: suricata.eve.dns.rrname
  target_field: dns.question.registered_domain
  target_subdomain_field: dns.question.subdomain
+ target_etld_field: dns.question.top_level_domain
  - add_fields:
  target: ''
  fields:

x-pack/filebeat/module/zeek/dns/config/dns.yml

@@ -20,6 +20,7 @@ processors:
  field: zeek.dns.query
  target_field: dns.question.registered_domain
  target_subdomain_field: dns.question.subdomain
+ target_etld_field: dns.question.top_level_domain
  - script:
  lang: javascript
  id: zeek_dns_flags

x-pack/winlogbeat/module/sysmon/config/winlogbeat-sysmon.js

@@ -1564,6 +1564,7 @@ var sysmon = (function () {
  field: "dns.question.name",
  target_field: "dns.question.registered_domain",
  target_subdomain_field: "dns.question.subdomain",
+ target_etld_field: "dns.question.top_level_domain",
  })
  .Add(setRuleName)
  .Add(translateDnsQueryStatus)