From c2f141ba107d082b1250701f57c8d7db87a9114e Mon Sep 17 00:00:00 2001
From: kaiyan-sheng <kaiyan.sheng@elastic.co>
Date: Tue, 1 Dec 2020 12:18:05 -0700
Subject: [PATCH] [RFC] Add host metric fields to ECS (stage 2) (#1028)

---
 rfcs/text/0005-host-metric-fields.md |  25 +++--
 rfcs/text/0005/ec2.json              | 150 +++++++++++++++++++++++++++
 rfcs/text/0005/host.yml              |  61 +++++++++++
 3 files changed, 223 insertions(+), 13 deletions(-)
 create mode 100644 rfcs/text/0005/ec2.json
 create mode 100644 rfcs/text/0005/host.yml

diff --git a/rfcs/text/0005-host-metric-fields.md b/rfcs/text/0005-host-metric-fields.md
index 64d964c482..f604a40008 100644
--- a/rfcs/text/0005-host-metric-fields.md
+++ b/rfcs/text/0005-host-metric-fields.md
@@ -1,8 +1,8 @@
 # 0005: Host Metric Fields
 <!-- Leave this ID at 0000. The ECS team will assign a unique, contiguous RFC number upon merging the initial stage of this RFC. -->
 
-- Stage: **1 (proposal)** <!-- Update to reflect target stage. See https://elastic.github.io/ecs/stages.html -->
-- Date: **2020-10-13** <!-- The ECS team sets this date at merge time. This is the date of the latest stage advancement. -->
+- Stage: **2 (draft)** <!-- Update to reflect target stage. See https://elastic.github.io/ecs/stages.html -->
+- Date: **TBD** <!-- The ECS team sets this date at merge time. This is the date of the latest stage advancement. -->
 
 <!--
 As you work on your RFC, use the "Stage N" comments to guide you in what you should focus on, for the stage you're targeting.
@@ -22,21 +22,16 @@ Proposed 7 new fields are:
 * host.disk.write.bytes
 
 ## Fields
+This RFC calls for the addition of host fields to collect basic monitoring metrics from a host or VM such as CPU, network and disk.
+Please see [`host`](0005/host.yml) for definitions of all fields.
+
+Note: the `host.network.*` and `host.disk.*` fields are gauges which represent
+IO since the last metric collection. In order to interpret these metrics, the
+collection period is needed. Collection period will be added into ECS separately.
 
 <!--
 Stage 1: Describe at a high level how this change affects fields. Which fieldsets will be impacted? How many fields overall? Are we primarily adding fields, removing fields, or changing existing fields? The goal here is to understand the fundamental technical implications and likely extent of these changes. ~2-5 sentences.
 -->
-This RFC calls for the addition of host fields to collect basic monitoring metrics from a host or VM such as CPU, network and disk.
-
-| field | type | description |
-| --- | --- | --- |
-| `host.cpu.usage` | scaled_float (scaling_factor 1000) | Percent CPU used with scaling_factor of 1000. This value is normalized by the number of CPU cores and it ranges from 0 to 1. For example: For a two core host, this value should be the average of the 2 cores, between 0 and 1. |
-| `host.network.ingress.bytes` | long | The number of bytes received (gauge) on all network interfaces by the host in a given period of time. |
-| `host.network.ingress.packets` | long | The number of packets (gauge) received on all network interfaces by the host in a given period of time. |
-| `host.network.egress.bytes` | long | The number of bytes (gauge) sent out on all network interfaces by the host in a given period of time. |
-| `host.network.egress.packets` | long | The number of packets (gauge) sent out on all network interfaces by the host in a given period of time. |
-| `host.disk.read.bytes` | long | The total number of bytes (gauge) read successfully in a given period of time. |
-| `host.disk.write.bytes` | long | The total number of bytes (gauge) write successfully in a given period of time. |
 
 <!--
 Stage 2: Include new or updated yml field definitions for all of the essential fields in this draft. While not exhaustive, the fields documented here should be comprehensive enough to deeply evaluate the technical considerations of this change. The goal here is to validate the technical details for all essential fields and to provide a basis for adding experimental field definitions to the schema. Use GitHub code blocks with yml syntax formatting.
@@ -74,6 +69,9 @@ Stage 1: Provide a high-level description of example sources of data. This does
 Stage 2: Included a real world example source document. Ideally this example comes from the source(s) identified in stage 1. If not, it should replace them. The goal here is to validate the utility of these field changes in the context of a real world example. Format with the source name as a ### header and the example document in a GitHub code block with json formatting.
 -->
 
+Please see example source document from AWS EC2 instance in
+[rfcs/text/0005/ec2.json](0005/ec2.json).
+
 <!--
 Stage 3: Add more real world example source documents so we have at least 2 total, but ideally 3. Format as described in stage 2.
 -->
@@ -159,6 +157,7 @@ e.g.:
 
 * Stage 0: https://github.com/elastic/ecs/pull/947
 * Stage 1: https://github.com/elastic/ecs/pull/950
+* Stage 2: https://github.com/elastic/ecs/pull/1028
 
 <!--
 * Stage 1: https://github.com/elastic/ecs/pull/NNN
diff --git a/rfcs/text/0005/ec2.json b/rfcs/text/0005/ec2.json
new file mode 100644
index 0000000000..52c553cc39
--- /dev/null
+++ b/rfcs/text/0005/ec2.json
@@ -0,0 +1,150 @@
+{
+  "_index": "metricbeat-8.0.0-2020.10.13-000001",
+  "_type": "_doc",
+  "_id": "7P6XKnUBnE_5JbVxdPGQ",
+  "_version": 1,
+  "_score": null,
+  "_source": {
+    "@timestamp": "2020-10-15T04:39:00.000Z",
+    "host": {
+      "id": "i-04c1a32c2aace6b40",
+      "name": "i-04c1a32c2aace6b40",
+      "cpu": {
+        "usage": 0.0020056497175091896
+      },
+      "disk": {
+        "write": {
+          "bytes": 0
+        },
+        "read": {
+          "bytes": 0
+        }
+      },
+      "network": {
+        "ingress": {
+          "bytes": 157612,
+          "packets": 2272
+        },
+        "egress": {
+          "bytes": 79944,
+          "packets": 1199
+        }
+      }
+    },
+    "event": {
+      "dataset": "aws.ec2",
+      "module": "aws",
+      "duration": 23277249413
+    },
+    "metricset": {
+      "name": "ec2",
+      "period": 300000
+    },
+    "service": {
+      "type": "aws"
+    },
+    "agent": {
+      "type": "metricbeat",
+      "version": "8.0.0",
+      "ephemeral_id": "f0733710-5558-4346-966d-28b4231e0765",
+      "id": "a8a83d26-7399-4bc3-bde6-a385aa3c14e4",
+      "name": "KaiyanMacBookPro"
+    },
+    "ecs": {
+      "version": "1.6.0"
+    },
+    "aws": {
+      "ec2": {
+        "instance": {
+          "image": {
+            "id": "ami-0b418580298265d5c"
+          },
+          "state": {
+            "code": 16,
+            "name": "running"
+          },
+          "monitoring": {
+            "state": "disabled"
+          },
+          "core": {
+            "count": 1
+          },
+          "threads_per_core": 1,
+          "public": {
+            "ip": "3.122.204.80",
+            "dns_name": ""
+          },
+          "private": {
+            "ip": "10.0.0.122",
+            "dns_name": "ip-10-0-0-122.eu-central-1.compute.internal"
+          }
+        },
+        "cpu": {
+          "credit_usage": 0.009044,
+          "credit_balance": 144,
+          "surplus_credit_balance": 0,
+          "surplus_credits_charged": 0,
+          "total": {
+            "pct": 0.20056497175091897
+          }
+        },
+        "status": {
+          "check_failed_system": 0,
+          "check_failed": 0,
+          "check_failed_instance": 0
+        },
+        "network": {
+          "in": {
+            "bytes": 157612,
+            "packets": 2272,
+            "bytes_per_sec": 525.3733333333333,
+            "packets_per_sec": 7.573333333333333
+          },
+          "out": {
+            "packets": 1199,
+            "bytes": 79944,
+            "bytes_per_sec": 266.48,
+            "packets_per_sec": 3.9966666666666666
+          }
+        },
+        "diskio": {
+          "write": {
+            "bytes": 0,
+            "count": 0,
+            "bytes_per_sec": 0,
+            "count_per_sec": 0
+          },
+          "read": {
+            "bytes": 0,
+            "count": 0,
+            "bytes_per_sec": 0,
+            "count_per_sec": 0
+          }
+        }
+      }
+    },
+    "cloud": {
+      "machine": {
+        "type": "t2.micro"
+      },
+      "availability_zone": "eu-central-1a",
+      "provider": "aws",
+      "region": "eu-central-1",
+      "account": {
+        "name": "elastic-beats",
+        "id": "428152502467"
+      },
+      "instance": {
+        "id": "i-04c1a32c2aace6b40"
+      }
+    }
+  },
+  "fields": {
+    "@timestamp": [
+        "2020-10-15T04:39:00.000Z"
+    ]
+  },
+  "sort": [
+      1602736740000
+  ]
+}
diff --git a/rfcs/text/0005/host.yml b/rfcs/text/0005/host.yml
new file mode 100644
index 0000000000..3382f2ed77
--- /dev/null
+++ b/rfcs/text/0005/host.yml
@@ -0,0 +1,61 @@
+- name: host
+  fields:
+    - name: cpu.usage
+      type: scaled_float
+      scaling_factor: 1000
+      level: extended
+      short: Percent CPU used, between 0 and 1.
+      description: >
+        Percent CPU used which is normalized by the number of CPU cores and it
+        ranges from 0 to 1. Scaling factor: 1000.
+
+        For example: For a two core host, this value should be the average of the
+        two cores, between 0 and 1.
+
+    - name: network.ingress.bytes
+      type: long
+      level: extended
+      short: The number of bytes received on all network interfaces.
+      description: >
+        The number of bytes received (gauge) on all network interfaces by the
+        host since the last metric collection.
+
+    - name: network.ingress.packets
+      type: long
+      level: extended
+      short: The number of packets received on all network interfaces.
+      description: >
+        The number of packets (gauge) received on all network interfaces by the
+        host since the last metric collection.
+
+    - name: network.egress.bytes
+      type: long
+      level: extended
+      short: The number of bytes sent on all network interfaces.
+      description: >
+        The number of bytes (gauge) sent out on all network interfaces by the
+        host since the last metric collection.
+
+    - name: network.egress.packets
+      type: long
+      level: extended
+      short: The number of packets sent on all network interfaces.
+      description: >
+        The number of packets (gauge) sent out on all network interfaces by the
+        host since the last metric collection.
+
+    - name: disk.read.bytes
+      type: long
+      level: extended
+      short: The number of bytes read by all disks.
+      description: >
+        The total number of bytes (gauge) read successfully (aggregated from all
+        disks) since the last metric collection.
+
+    - name: disk.write.bytes
+      type: long
+      level: extended
+      short: The number of bytes written on all disks.
+      description: >
+        The total number of bytes (gauge) written successfully (aggregated from
+        all disks) since the last metric collection.