Add email SMTP envelope fields

CHANGELOG.next.md

@@ -16,6 +16,8 @@ Thanks, you're awesome :-) -->
 
 #### Added
 
+* Add email SMTP transmission fields for "MAIL FROM", "RCPT TO". #2413
+
 #### Improvements
 
 * Define base encoding of `x509.serial_number`. #2383

docs/fields/field-details.asciidoc

@@ -3102,6 +3102,28 @@ example: `c26dbea0-80d5-463b-b93c-4e8b708219ce`
 
 // ===============================================================
 
+|
+[[field-email-mailfrom-address]]
+<<field-email-mailfrom-address, email.mailfrom.address>>
+
+a| beta:[ This field is beta and subject to change. ]
+
+The address from the SMTP `MAIL` command associated with the email, at the observed email relay host.
+
+This is commonly called `MAIL FROM` value of the SMTP envelope.
+
+For example, if the SMTP command is `MAIL FROM:<Smith@bar.com>`, this field has a value of `Smith@bar.com`
+
+type: keyword
+
+
+
+example: `userx@y.foo.org`
+
+| extended
+
+// ===============================================================
+
 |
 [[field-email-message-id]]
 <<field-email-message-id, email.message_id>>
@@ -3134,6 +3156,31 @@ example: `2020-11-10T22:12:34.8196921Z`
 
 // ===============================================================
 
+|
+[[field-email-recipient-addresses]]
+<<field-email-recipient-addresses, email.recipient.addresses>>
+
+a| beta:[ This field is beta and subject to change. ]
+
+The email address(es) from SMTP `RCPT` command(s) associated with the email message.
+
+This is commonly called `RCPT TO` value of the SMTP envelope.
+
+As there may be multiple `RCPT` commands during a single message transaction, this may contain all email address from the entire transaction.
+
+type: keyword
+
+
+Note: this field should contain an array of values.
+
+
+
+example: `["Jones@foo.com", "Green@foo.com", "Brown@foo.com"]`
+
+| extended
+
+// ===============================================================
+
 |
 [[field-email-reply-to-address]]
 <<field-email-reply-to-address, email.reply_to.address>>

experimental/generated/beats/fields.ecs.yml

@@ -1914,6 +1914,19 @@
         Identifier is not persistent across hops.'
       example: c26dbea0-80d5-463b-b93c-4e8b708219ce
       default_field: false
+    - name: mailfrom.address
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: 'The address from the SMTP `MAIL` command associated with the email,
+        at the observed email relay host.
+
+        This is commonly called `MAIL FROM` value of the SMTP envelope.
+
+        For example, if the SMTP command is `MAIL FROM:<Smith@bar.com>`, this field
+        has a value of `Smith@bar.com`'
+      example: userx@y.foo.org
+      default_field: false
     - name: message_id
       level: extended
       type: wildcard
@@ -1928,6 +1941,19 @@
         will fill in this value automatically when the message is sent by a user.
       example: '2020-11-10T22:12:34.8196921Z'
       default_field: false
+    - name: recipient.addresses
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: 'The email address(es) from SMTP `RCPT` command(s) associated with
+        the email message.
+
+        This is commonly called `RCPT TO` value of the SMTP envelope.
+
+        As there may be multiple `RCPT` commands during a single message transaction,
+        this may contain all email address from the entire transaction.'
+      example: '["Jones@foo.com", "Green@foo.com", "Brown@foo.com"]'
+      default_field: false
     - name: reply_to.address
       level: extended
       type: keyword

experimental/generated/csv/fields.csv

@@ -229,8 +229,10 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description
 9.0.0-dev+exp,true,email,email.direction,keyword,extended,,inbound,Direction of the message.
 9.0.0-dev+exp,true,email,email.from.address,keyword,extended,array,sender@example.com,The sender's email address.
 9.0.0-dev+exp,true,email,email.local_id,keyword,extended,,c26dbea0-80d5-463b-b93c-4e8b708219ce,Unique identifier given by the source.
+9.0.0-dev+exp,true,email,email.mailfrom.address,keyword,extended,,userx@y.foo.org,Email address from SMTP `MAIL` command.
 9.0.0-dev+exp,true,email,email.message_id,wildcard,extended,,81ce15$8r2j59@mail01.example.com,Value from the Message-ID header.
 9.0.0-dev+exp,true,email,email.origination_timestamp,date,extended,,2020-11-10T22:12:34.8196921Z,Date and time the email was composed.
+9.0.0-dev+exp,true,email,email.recipient.addresses,keyword,extended,array,"[""Jones@foo.com"", ""Green@foo.com"", ""Brown@foo.com""]",Recipient addresses from the SMTP `RCPT` commands.
 9.0.0-dev+exp,true,email,email.reply_to.address,keyword,extended,array,reply.here@example.com,Address replies should be delivered to.
 9.0.0-dev+exp,true,email,email.sender.address,keyword,extended,,,Address of the message sender.
 9.0.0-dev+exp,true,email,email.subject,keyword,extended,,Please see this important message.,The subject of the email message.

experimental/generated/ecs/ecs_flat.yml

@@ -2819,6 +2819,24 @@ email.local_id:
   normalize: []
   short: Unique identifier given by the source.
   type: keyword
+email.mailfrom.address:
+  beta: This field is beta and subject to change.
+  dashed_name: email-mailfrom-address
+  description: 'The address from the SMTP `MAIL` command associated with the email,
+    at the observed email relay host.
+
+    This is commonly called `MAIL FROM` value of the SMTP envelope.
+
+    For example, if the SMTP command is `MAIL FROM:<Smith@bar.com>`, this field has
+    a value of `Smith@bar.com`'
+  example: userx@y.foo.org
+  flat_name: email.mailfrom.address
+  ignore_above: 1024
+  level: extended
+  name: mailfrom.address
+  normalize: []
+  short: Email address from SMTP `MAIL` command.
+  type: keyword
 email.message_id:
   dashed_name: email-message-id
   description: Identifier from the RFC 5322 `Message-ID:` email header that refers
@@ -2841,6 +2859,25 @@ email.origination_timestamp:
   normalize: []
   short: Date and time the email was composed.
   type: date
+email.recipient.addresses:
+  beta: This field is beta and subject to change.
+  dashed_name: email-recipient-addresses
+  description: 'The email address(es) from SMTP `RCPT` command(s) associated with
+    the email message.
+
+    This is commonly called `RCPT TO` value of the SMTP envelope.
+
+    As there may be multiple `RCPT` commands during a single message transaction,
+    this may contain all email address from the entire transaction.'
+  example: '["Jones@foo.com", "Green@foo.com", "Brown@foo.com"]'
+  flat_name: email.recipient.addresses
+  ignore_above: 1024
+  level: extended
+  name: recipient.addresses
+  normalize:
+  - array
+  short: Recipient addresses from the SMTP `RCPT` commands.
+  type: keyword
 email.reply_to.address:
   dashed_name: email-reply-to-address
   description: The address that replies should be delivered to based on the value

experimental/generated/ecs/ecs_nested.yml

@@ -3785,6 +3785,24 @@ email:
       normalize: []
       short: Unique identifier given by the source.
       type: keyword
+    email.mailfrom.address:
+      beta: This field is beta and subject to change.
+      dashed_name: email-mailfrom-address
+      description: 'The address from the SMTP `MAIL` command associated with the email,
+        at the observed email relay host.
+
+        This is commonly called `MAIL FROM` value of the SMTP envelope.
+
+        For example, if the SMTP command is `MAIL FROM:<Smith@bar.com>`, this field
+        has a value of `Smith@bar.com`'
+      example: userx@y.foo.org
+      flat_name: email.mailfrom.address
+      ignore_above: 1024
+      level: extended
+      name: mailfrom.address
+      normalize: []
+      short: Email address from SMTP `MAIL` command.
+      type: keyword
     email.message_id:
       dashed_name: email-message-id
       description: Identifier from the RFC 5322 `Message-ID:` email header that refers
@@ -3807,6 +3825,25 @@ email:
       normalize: []
       short: Date and time the email was composed.
       type: date
+    email.recipient.addresses:
+      beta: This field is beta and subject to change.
+      dashed_name: email-recipient-addresses
+      description: 'The email address(es) from SMTP `RCPT` command(s) associated with
+        the email message.
+
+        This is commonly called `RCPT TO` value of the SMTP envelope.
+
+        As there may be multiple `RCPT` commands during a single message transaction,
+        this may contain all email address from the entire transaction.'
+      example: '["Jones@foo.com", "Green@foo.com", "Brown@foo.com"]'
+      flat_name: email.recipient.addresses
+      ignore_above: 1024
+      level: extended
+      name: recipient.addresses
+      normalize:
+      - array
+      short: Recipient addresses from the SMTP `RCPT` commands.
+      type: keyword
     email.reply_to.address:
       dashed_name: email-reply-to-address
       description: The address that replies should be delivered to based on the value

experimental/generated/elasticsearch/composable/component/email.json

@@ -107,12 +107,28 @@
               "ignore_above": 1024,
               "type": "keyword"
             },
+            "mailfrom": {
+              "properties": {
+                "address": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                }
+              }
+            },
             "message_id": {
               "type": "wildcard"
             },
             "origination_timestamp": {
               "type": "date"
             },
+            "recipient": {
+              "properties": {
+                "addresses": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                }
+              }
+            },
             "reply_to": {
               "properties": {
                 "address": {

experimental/generated/elasticsearch/legacy/template.json

@@ -1151,12 +1151,28 @@
             "ignore_above": 1024,
             "type": "keyword"
           },
+          "mailfrom": {
+            "properties": {
+              "address": {
+                "ignore_above": 1024,
+                "type": "keyword"
+              }
+            }
+          },
           "message_id": {
             "type": "wildcard"
           },
           "origination_timestamp": {
             "type": "date"
           },
+          "recipient": {
+            "properties": {
+              "addresses": {
+                "ignore_above": 1024,
+                "type": "keyword"
+              }
+            }
+          },
           "reply_to": {
             "properties": {
               "address": {

generated/beats/fields.ecs.yml

@@ -1864,6 +1864,19 @@
         Identifier is not persistent across hops.'
       example: c26dbea0-80d5-463b-b93c-4e8b708219ce
       default_field: false
+    - name: mailfrom.address
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: 'The address from the SMTP `MAIL` command associated with the email,
+        at the observed email relay host.
+
+        This is commonly called `MAIL FROM` value of the SMTP envelope.
+
+        For example, if the SMTP command is `MAIL FROM:<Smith@bar.com>`, this field
+        has a value of `Smith@bar.com`'
+      example: userx@y.foo.org
+      default_field: false
     - name: message_id
       level: extended
       type: wildcard
@@ -1878,6 +1891,19 @@
         will fill in this value automatically when the message is sent by a user.
       example: '2020-11-10T22:12:34.8196921Z'
       default_field: false
+    - name: recipient.addresses
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: 'The email address(es) from SMTP `RCPT` command(s) associated with
+        the email message.
+
+        This is commonly called `RCPT TO` value of the SMTP envelope.
+
+        As there may be multiple `RCPT` commands during a single message transaction,
+        this may contain all email address from the entire transaction.'
+      example: '["Jones@foo.com", "Green@foo.com", "Brown@foo.com"]'
+      default_field: false
     - name: reply_to.address
       level: extended
       type: keyword

generated/csv/fields.csv

@@ -222,8 +222,10 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description
 9.0.0-dev,true,email,email.direction,keyword,extended,,inbound,Direction of the message.
 9.0.0-dev,true,email,email.from.address,keyword,extended,array,sender@example.com,The sender's email address.
 9.0.0-dev,true,email,email.local_id,keyword,extended,,c26dbea0-80d5-463b-b93c-4e8b708219ce,Unique identifier given by the source.
+9.0.0-dev,true,email,email.mailfrom.address,keyword,extended,,userx@y.foo.org,Email address from SMTP `MAIL` command.
 9.0.0-dev,true,email,email.message_id,wildcard,extended,,81ce15$8r2j59@mail01.example.com,Value from the Message-ID header.
 9.0.0-dev,true,email,email.origination_timestamp,date,extended,,2020-11-10T22:12:34.8196921Z,Date and time the email was composed.
+9.0.0-dev,true,email,email.recipient.addresses,keyword,extended,array,"[""Jones@foo.com"", ""Green@foo.com"", ""Brown@foo.com""]",Recipient addresses from the SMTP `RCPT` commands.
 9.0.0-dev,true,email,email.reply_to.address,keyword,extended,array,reply.here@example.com,Address replies should be delivered to.
 9.0.0-dev,true,email,email.sender.address,keyword,extended,,,Address of the message sender.
 9.0.0-dev,true,email,email.subject,keyword,extended,,Please see this important message.,The subject of the email message.

generated/ecs/ecs_flat.yml

@@ -2750,6 +2750,24 @@ email.local_id:
   normalize: []
   short: Unique identifier given by the source.
   type: keyword
+email.mailfrom.address:
+  beta: This field is beta and subject to change.
+  dashed_name: email-mailfrom-address
+  description: 'The address from the SMTP `MAIL` command associated with the email,
+    at the observed email relay host.
+
+    This is commonly called `MAIL FROM` value of the SMTP envelope.
+
+    For example, if the SMTP command is `MAIL FROM:<Smith@bar.com>`, this field has
+    a value of `Smith@bar.com`'
+  example: userx@y.foo.org
+  flat_name: email.mailfrom.address
+  ignore_above: 1024
+  level: extended
+  name: mailfrom.address
+  normalize: []
+  short: Email address from SMTP `MAIL` command.
+  type: keyword
 email.message_id:
   dashed_name: email-message-id
   description: Identifier from the RFC 5322 `Message-ID:` email header that refers
@@ -2772,6 +2790,25 @@ email.origination_timestamp:
   normalize: []
   short: Date and time the email was composed.
   type: date
+email.recipient.addresses:
+  beta: This field is beta and subject to change.
+  dashed_name: email-recipient-addresses
+  description: 'The email address(es) from SMTP `RCPT` command(s) associated with
+    the email message.
+
+    This is commonly called `RCPT TO` value of the SMTP envelope.
+
+    As there may be multiple `RCPT` commands during a single message transaction,
+    this may contain all email address from the entire transaction.'
+  example: '["Jones@foo.com", "Green@foo.com", "Brown@foo.com"]'
+  flat_name: email.recipient.addresses
+  ignore_above: 1024
+  level: extended
+  name: recipient.addresses
+  normalize:
+  - array
+  short: Recipient addresses from the SMTP `RCPT` commands.
+  type: keyword
 email.reply_to.address:
   dashed_name: email-reply-to-address
   description: The address that replies should be delivered to based on the value