[Metricbeat] Add Load Balancing metricset to Google Cloud Platform module

CHANGELOG.next.asciidoc

@@ -154,6 +154,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Reuse connections in SQL module. {pull}16001[16001]
 - Improve the `logstash` module (when `xpack.enabled` is set to `true`) to use the override `cluster_uuid` returned by Logstash APIs. {issue}15772[15772] {pull}15795[15795]
 - Add kubernetes storage class support via kube-state-metrics. {pull}16145[16145]
+- Add Load Balancing metricset to GCP {pull}15559[15559]
 
 *Packetbeat*
 

metricbeat/docs/modules/googlecloud.asciidoc

@@ -96,5 +96,9 @@ The following metricsets are available:
 
 * <<metricbeat-metricset-googlecloud-compute,compute>>
 
+* <<metricbeat-metricset-googlecloud-loadbalancing,loadbalancing>>
+
 include::googlecloud/compute.asciidoc[]
 
+include::googlecloud/loadbalancing.asciidoc[]
+

metricbeat/docs/modules/googlecloud/loadbalancing.asciidoc

@@ -0,0 +1,23 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-googlecloud-loadbalancing]]
+=== Google Cloud Platform loadbalancing metricset
+
+beta[]
+
+include::../../../../x-pack/metricbeat/module/googlecloud/loadbalancing/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-googlecloud,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../../x-pack/metricbeat/module/googlecloud/loadbalancing/_meta/data.json[]
+----

metricbeat/docs/modules_list.asciidoc

@@ -92,7 +92,8 @@ This file is generated! See scripts/mage/docs_collector.go
 .2+| .2+| |<<metricbeat-metricset-golang-expvar,expvar>> 
 |<<metricbeat-metricset-golang-heap,heap>> 
 |<<metricbeat-module-googlecloud,Google Cloud Platform>> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | 
-.1+| .1+| |<<metricbeat-metricset-googlecloud-compute,compute>> beta[] 
+.2+| .2+| |<<metricbeat-metricset-googlecloud-compute,compute>> beta[] 
+|<<metricbeat-metricset-googlecloud-loadbalancing,loadbalancing>> beta[] 
 |<<metricbeat-module-graphite,Graphite>> |image:./images/icon-no.png[No prebuilt dashboards] | 
 .1+| .1+| |<<metricbeat-metricset-graphite-server,server>> 
 |<<metricbeat-module-haproxy,HAProxy>> |image:./images/icon-yes.png[Prebuilt dashboards are available] | 

x-pack/metricbeat/module/googlecloud/constants.go

@@ -18,10 +18,11 @@ const (
 
 // Metricsets / GCP services names
 const (
- ServiceCompute = "compute"
- ServicePubsub = "pubsub"
- ServiceFirestore = "firestore"
- ServiceStorage = "storage"
+ ServiceCompute = "compute"
+ ServicePubsub = "pubsub"
+ ServiceLoadBalancing = "loadbalancing"
+ ServiceFirestore = "firestore"
+ ServiceStorage = "storage"
 )
 
 //Paths within the GCP monitoring.TimeSeries response, if converted to JSON, where you can find each ECS field required for the output event

x-pack/metricbeat/module/googlecloud/loadbalancing/_meta/data.json

@@ -0,0 +1,174 @@
+{
+ "@timestamp": "2020-01-14T21:14:00.000Z",
+ "@metadata": {
+ "beat": "metricbeat",
+ "type": "_doc",
+ "version": "8.0.0"
+ },
+ "host": {
+ "name": "mcastro",
+ "architecture": "x86_64",
+ "os": {
+ "version": "",
+ "family": "",
+ "name": "Antergos Linux",
+ "kernel": "5.4.3-arch1-1",
+ "platform": "antergos"
+ },
+ "id": "54f70115bae545cbac2b150f254472a0",
+ "containerized": false,
+ "hostname": "mcastro"
+ },
+ "agent": {
+ "type": "metricbeat",
+ "ephemeral_id": "ddb5f424-578e-4c51-ac74-08447827a4b1",
+ "hostname": "mcastro",
+ "id": "7e36a073-1a32-4a94-b65b-4c7f971fb228",
+ "version": "8.0.0"
+ },
+ "service": {
+ "type": "googlecloud"
+ },
+ "cloud": {
+ "account": {
+ "id": "elastic-metricbeat"
+ },
+ "provider": "googlecloud"
+ },
+ "event": {
+ "dataset": "googlecloud.loadbalancing",
+ "module": "googlecloud",
+ "duration": 908872754
+ },
+ "metricset": {
+ "name": "loadbalancing",
+ "period": 900000
+ },
+ "googlecloud": {
+ "labels": {
+ "metrics": {
+ "response_code": "502",
+ "cache_result": "DISABLED",
+ "proxy_continent": "Europe",
+ "client_country": "France",
+ "response_code_class": "500",
+ "protocol": "HTTP/1.1"
+ },
+ "resource": {
+ "matched_url_path_rule": "UNMATCHED",
+ "backend_target_name": "my-backed",
+ "backend_type": "INVALID_BACKEND",
+ "backend_scope_type": "INVALID_BACKEND",
+ "url_map_name": "test-lb",
+ "forwarding_rule_name": "test-lb-forwarding-rule",
+ "region": "global",
+ "backend_scope": "INVALID_BACKEND",
+ "target_proxy_name": "test-lb-target-proxy",
+ "backend_target_type": "BACKEND_SERVICE",
+ "backend_name": "INVALID_BACKEND"
+ }
+ },
+ "loadbalancing": {
+ "https": {
+ "frontend_tcp_rtt": {
+ "count": 3,
+ "mean": 27,
+ "bucket_options": {
+ "Options": {
+ "ExponentialBuckets": {
+ "growth_factor": 1.4,
+ "scale": 1,
+ "num_finite_buckets": 66
+ }
+ }
+ },
+ "bucket_counts": [
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 3
+ ]
+ },
+ "request_bytes_count": 1281,
+ "request_count": 3,
+ "response_bytes_count": 1464,
+ "backend_latencies": {
+ "mean": 112.094,
+ "bucket_options": {
+ "Options": {
+ "ExponentialBuckets": {
+ "growth_factor": 1.4,
+ "scale": 1,
+ "num_finite_buckets": 66
+ }
+ }
+ },
+ "bucket_counts": [
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 3
+ ],
+ "count": 3
+ },
+ "total_latencies": {
+ "bucket_counts": [
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 3
+ ],
+ "count": 3,
+ "mean": 195,
+ "bucket_options": {
+ "Options": {
+ "ExponentialBuckets": {
+ "growth_factor": 1.4,
+ "scale": 1,
+ "num_finite_buckets": 66
+ }
+ }
+ }
+ },
+ "backend_request_count": 3,
+ "backend_response_bytes_count": 1464,
+ "backend_request_bytes_count": 1281
+ }
+ }
+ },
+ "ecs": {
+ "version": "1.4.0"
+ }
+}

x-pack/metricbeat/module/googlecloud/loadbalancing/_meta/docs.asciidoc

@@ -0,0 +1,27 @@
+Load Balancing metricset to fetch metrics from https://cloud.google.com/load-balancing/[Load Balancing] in Google Cloud Platform.
+
+The `loadbalancing` metricset contains all metrics exported from the https://cloud.google.com/monitoring/api/metrics_gcp#gcp-loadbalancing[Stackdriver API]. The field names have been left untouched for people already familiar with them.
+
+[float]
+=== Fields
+
+- `loadbalancing.https.backend_latencies`: A distribution of the latency calculated from when the request was sent by the proxy to the backend until the proxy received from the backend the last byte of response.
+- `loadbalancing.https.backend_request_bytes_count`: The number of bytes sent as requests from HTTP/S load balancer to backends.
+- `loadbalancing.https.backend_request_count`: The number of requests served by backends of HTTP/S load balancer.
+- `loadbalancing.https.backend_response_bytes_count`: The number of bytes sent as responses from backends (or cache) to HTTP/S load balancer.
+- `loadbalancing.https.frontend_tcp_rtt`: A distribution of the RTT measured for each connection between client and proxy.
+- `loadbalancing.https.request_bytes_count`: The number of bytes sent as requests from clients to HTTP/S load balancer.
+- `loadbalancing.https.request_count`: The number of requests served by HTTP/S load balancer.
+- `loadbalancing.https.response_bytes_count`: The number of bytes sent as responses from HTTP/S load balancer to clients.
+- `loadbalancing.https.total_latencies`: A distribution of the latency calculated from when the request was received by the proxy until the proxy got ACK from client on last response byte.
+- `loadbalancing.l3.internal.egress_bytes_count`: The number of bytes sent from ILB backend to client (for TCP flows it's counting bytes on application stream only).
+- `loadbalancing.l3.internal.egress_packets_count`: The number of packets sent from ILB backend to client of the flow.
+- `loadbalancing.l3.internal.ingress_bytes_count`: The number of bytes sent from client to ILB backend (for TCP flows it's counting bytes on application stream only).
+- `loadbalancing.l3.internal.ingress_packets_count`: The number of packets sent from client to ILB backend.
+- `loadbalancing.l3.internal.rtt_latencies`: A distribution of RTT measured over TCP connections for ILB flows.
+- `loadbalancing.tcp_ssl_proxy.closed_connections`: Number of connections that were terminated over TCP/SSL proxy.
+- `loadbalancing.tcp_ssl_proxy.egress_bytes_count`: Number of bytes sent from VM to client using proxy.
+- `loadbalancing.tcp_ssl_proxy.frontend_tcp_rtt`: A distribution of the smoothed RTT (in ms) measured by the proxy's TCP stack, each minute application layer bytes pass from proxy to client.
+- `loadbalancing.tcp_ssl_proxy.ingress_bytes_count`: Number of bytes sent from client to VM using proxy.
+- `loadbalancing.tcp_ssl_proxy.new_connections`: Number of connections that were created over TCP/SSL proxy.
+- `loadbalancing.tcp_ssl_proxy.open_connections`: Current number of outstanding connections through the TCP/SSL proxy.