Cherry-pick #18802 to 7.x: Add memory metrics into compute googlecloud (

#18831) * Add memory metrics into compute googlecloud (#18802) * Add memory metrics into compute googlecloud * update unit test and data.json for memory metrics (cherry picked from commit 1eb4e79) * update changelog
elastic · May 29, 2020 · c9a6f9a · c9a6f9a
1 parent f4e97c5
commit c9a6f9a
Show file tree

Hide file tree

Showing 14 changed files with 274 additions and 59 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -243,6 +243,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Remove required for region/zone and make stackdriver a metricset in googlecloud. {issue}16785[16785] {pull}18398[18398]
 - Fix application_pool metricset after pdh changes. {pull}18477[18477]
 - Fix tags_filter for cloudwatch metricset in aws. {pull}18524[18524]
+- Add missing network.sent_packets_count metric into compute metricset in googlecloud module. {pull}18802[18802]
 
 *Packetbeat*
 
@@ -515,6 +516,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Remove requirement to connect as sysdba in Oracle module {issue}15846[15846] {pull}18182[18182]
 - Update MSSQL module to fix some SSPI authentication and add brackets to USE statements {pull}17862[17862]]
 - Add client address to events from http server module {pull}18336[18336]
+- Add memory metrics into compute googlecloud. {pull}18802[18802]
 
 *Packetbeat*
 

diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -18230,6 +18230,9 @@ GCP module
 *`googlecloud.labels`*::
 +
 --
+Google cloud monitoring metrics labels
+
+
 type: object
 
 --
@@ -18381,6 +18384,43 @@ type: long
 
 --
 
+
+*`googlecloud.compute.instance.memory.balloon.ram_size.value`*::
++
+--
+The total amount of memory in the VM. This metric is only available for VMs that belong to the e2 family.
+
+type: long
+
+--
+
+*`googlecloud.compute.instance.memory.balloon.ram_used.value`*::
++
+--
+Memory currently used in the VM. This metric is only available for VMs that belong to the e2 family.
+
+type: long
+
+--
+
+*`googlecloud.compute.instance.memory.balloon.swap_in_bytes_count.value`*::
++
+--
+The amount of memory read into the guest from its own swap space. This metric is only available for VMs that belong to the e2 family.
+
+type: long
+
+--
+
+*`googlecloud.compute.instance.memory.balloon.swap_out_bytes_count.value`*::
++
+--
+The amount of memory written from the guest to its own swap space. This metric is only available for VMs that belong to the e2 family.
+
+type: long
+
+--
+
 [float]
 === loadbalancing
 

diff --git a/x-pack/metricbeat/module/googlecloud/_meta/fields.yml b/x-pack/metricbeat/module/googlecloud/_meta/fields.yml
@@ -9,15 +9,21 @@
  fields:
  - name: labels
  type: object
+ description: >
+ Google cloud monitoring metrics labels
  fields:
  - name: user.*
  type: object
+ object_type: keyword
  - name: metadata.*
  type: object
+ object_type: keyword
  - name: metrics.*
  type: object
+ object_type: keyword
  - name: system.*
  type: object
+ object_type: keyword
  - name: "stackdriver.*.*.*.*"
  type: object
  object_type: double

diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/data.json b/x-pack/metricbeat/module/googlecloud/compute/_meta/data.json
@@ -5,47 +5,66 @@
  "id": "elastic-observability"
  },
  "instance": {
- "id": "1174463293187628268",
- "name": "gke-observability-8--observability-8--bc1afd95-ngmh"
+ "id": "7692260917184259934",
+ "name": "instance-test-ks"
  },
  "machine": {
- "type": "n1-standard-4"
+ "type": "e2-standard-2"
  },
  "provider": "googlecloud"
  },
- "cloud.availability_zone": "europe-west1-c",
+ "cloud.availability_zone": "us-central1-a",
  "event": {
  "dataset": "googlecloud.compute",
  "duration": 115000,
  "module": "googlecloud"
  },
  "googlecloud": {
  "compute": {
+ "firewall": {
+ "dropped_bytes_count": {
+ "value": 171
+ },
+ "dropped_packets_count": {
+ "value": 4
+ }
+ },
  "instance": {
- "disk": {
- "read_bytes_count": {
- "value": 0
- },
- "read_ops_count": {
- "value": 0
+ "cpu": {
+ "reserved_cores": {
+ "value": 2
  },
- "write_bytes_count": {
- "value": 0
+ "usage_time": {
+ "value": 0.1803964574089818
  },
- "write_ops_count": {
- "value": 0
+ "utilization": {
+ "value": 0.001503303811741515
  }
+ },
+ "memory": {
+ "balloon": {
+ "ram_size": {
+ "value": 7896264704
+ },
+ "ram_used": {
+ "value": 416518144
+ },
+ "swap_in_bytes_count": {
+ "value": 0
+ },
+ "swap_out_bytes_count": {
+ "value": 0
+ }
+ }
+ },
+ "uptime": {
+ "value": 60.000000000000455
  }
  }
  },
  "labels": {
- "metrics": {
- "device_name": "disk-2",
- "device_type": "permanent",
- "storage_type": "pd-standard"
- },
  "user": {
- "goog-gke-node": ""
+ "created-by": "ks"
  }
  }
  },

diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_cpu.json b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_cpu.json
@@ -5,15 +5,15 @@
  "id": "elastic-observability"
  },
  "instance": {
- "id": "1174463293187628268",
- "name": "gke-observability-8--observability-8--bc1afd95-ngmh"
+ "id": "7692260917184259934",
+ "name": "instance-test-ks"
  },
  "machine": {
- "type": "n1-standard-4"
+ "type": "e2-standard-2"
  },
  "provider": "googlecloud"
  },
- "cloud.availability_zone": "europe-west1-c",
+ "cloud.availability_zone": "us-central1-a",
  "event": {
  "dataset": "googlecloud.compute",
  "duration": 115000,
@@ -23,32 +23,48 @@
  "compute": {
  "firewall": {
  "dropped_bytes_count": {
- "value": 181
+ "value": 171
  },
  "dropped_packets_count": {
- "value": 3
+ "value": 4
  }
  },
  "instance": {
  "cpu": {
  "reserved_cores": {
- "value": 4
+ "value": 2
  },
  "usage_time": {
- "value": 63.478293027728796
+ "value": 0.1803964574089818
  },
  "utilization": {
- "value": 0.26449288761553663
+ "value": 0.001503303811741515
+ }
+ },
+ "memory": {
+ "balloon": {
+ "ram_size": {
+ "value": 7896264704
+ },
+ "ram_used": {
+ "value": 416518144
+ },
+ "swap_in_bytes_count": {
+ "value": 0
+ },
+ "swap_out_bytes_count": {
+ "value": 0
+ }
  }
  },
  "uptime": {
- "value": 60
+ "value": 60.000000000000455
  }
  }
  },
  "labels": {
  "user": {
- "goog-gke-node": ""
+ "created-by": "ks"
  }
  }
  },

diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_disk.json b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_disk.json
@@ -5,15 +5,15 @@
  "id": "elastic-observability"
  },
  "instance": {
- "id": "8390997210852978465",
- "name": "gke-observability-7--observability-7--3dd3e39b-0jm5"
+ "id": "7692260917184259934",
+ "name": "instance-test-ks"
  },
  "machine": {
- "type": "n1-standard-4"
+ "type": "e2-standard-2"
  },
  "provider": "googlecloud"
  },
- "cloud.availability_zone": "europe-west1-c",
+ "cloud.availability_zone": "us-central1-a",
  "event": {
  "dataset": "googlecloud.compute",
  "duration": 115000,
@@ -30,22 +30,22 @@
  "value": 0
  },
  "write_bytes_count": {
- "value": 0
+ "value": 158506
  },
  "write_ops_count": {
- "value": 0
+ "value": 25
  }
  }
  }
  },
  "labels": {
  "metrics": {
- "device_name": "gke-observability-7-1--pvc-65581044-7d5d-11ea-8cd9-42010af0011c",
+ "device_name": "instance-test-ks",
  "device_type": "permanent",
  "storage_type": "pd-standard"
  },
  "user": {
- "goog-gke-node": ""
+ "created-by": "ks"
  }
  }
  },

diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_firewall.json b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_firewall.json
@@ -5,15 +5,15 @@
  "id": "elastic-observability"
  },
  "instance": {
- "id": "2528596280375797115",
- "name": "gke-dev-next-oblt-dev-next-oblt-pool-404d7f0c-cpj6"
+ "id": "7692260917184259934",
+ "name": "instance-test-ks"
  },
  "machine": {
- "type": "n1-standard-4"
+ "type": "e2-standard-2"
  },
  "provider": "googlecloud"
  },
- "cloud.availability_zone": "europe-west1-c",
+ "cloud.availability_zone": "us-central1-a",
  "event": {
  "dataset": "googlecloud.compute",
  "duration": 115000,
@@ -23,22 +23,38 @@
  "compute": {
  "firewall": {
  "dropped_bytes_count": {
- "value": 386
+ "value": 277
  },
  "dropped_packets_count": {
- "value": 7
+ "value": 5
  }
  },
  "instance": {
  "cpu": {
  "reserved_cores": {
- "value": 4
+ "value": 2
  },
  "usage_time": {
- "value": 106.88293868489563
+ "value": 0.18404532833979204
  },
  "utilization": {
- "value": 0.4453455778537318
+ "value": 0.001533711069498267
+ }
+ },
+ "memory": {
+ "balloon": {
+ "ram_size": {
+ "value": 7896264704
+ },
+ "ram_used": {
+ "value": 416260096
+ },
+ "swap_in_bytes_count": {
+ "value": 0
+ },
+ "swap_out_bytes_count": {
+ "value": 0
+ }
  }
  },
  "uptime": {
@@ -48,7 +64,7 @@
  },
  "labels": {
  "user": {
- "goog-gke-node": ""
+ "created-by": "ks"
  }
  }
  },