Migrate sqs metricset to use cloudwatch input as light weight module (#…

…26117)

CHANGELOG.next.asciidoc

@@ -939,6 +939,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add additional network metrics to docker/network {pull}25354[25354]
 - Migrate ec2 metricsets to use cloudwatch input. {pull}25924[25924]
 - Reduce number of requests done by kubernetes metricsets to kubelet. {pull}25782[25782]
+- Migrate sqs metricsets to use cloudwatch input. {pull}26117[26117]
 
 *Packetbeat*
 

metricbeat/docs/modules/aws/sqs.asciidoc

@@ -8,6 +8,7 @@ This file is generated! See scripts/mage/docs_collector.go
 
 include::../../../../x-pack/metricbeat/module/aws/sqs/_meta/docs.asciidoc[]
 
+This is a default metricset. If the host module is unconfigured, this metricset is enabled by default.
 
 ==== Fields
 

x-pack/metricbeat/module/aws/cloudwatch/metadata.go

@@ -9,18 +9,22 @@ import (
 
  "github.com/elastic/beats/v7/metricbeat/mb"
  "github.com/elastic/beats/v7/x-pack/metricbeat/module/aws/cloudwatch/ec2"
+ "github.com/elastic/beats/v7/x-pack/metricbeat/module/aws/cloudwatch/sqs"
 )
 
 // AWS namespaces
 const (
  namespaceEC2 = "AWS/EC2"
+ namespaceSQS = "AWS/SQS"
 )
 
 // addMetadata adds metadata to the given events map based on namespace
 func addMetadata(namespace string, endpoint string, regionName string, awsConfig awssdk.Config, events map[string]mb.Event) map[string]mb.Event {
  switch namespace {
  case namespaceEC2:
  return ec2.AddMetadata(endpoint, regionName, awsConfig, events)
+ case namespaceSQS:
+ return sqs.AddMetadata(endpoint, regionName, awsConfig, events)
  default:
  return events
  }

x-pack/metricbeat/module/aws/cloudwatch/sqs/metadata.go

@@ -0,0 +1,58 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package sqs
+
+import (
+ "context"
+ "fmt"
+ "strings"
+
+ awssdk "github.com/aws/aws-sdk-go-v2/aws"
+ "github.com/aws/aws-sdk-go-v2/service/sqs"
+ "github.com/aws/aws-sdk-go-v2/service/sqs/sqsiface"
+ "github.com/pkg/errors"
+
+ "github.com/elastic/beats/v7/libbeat/logp"
+ "github.com/elastic/beats/v7/metricbeat/mb"
+ awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws"
+)
+
+const metadataPrefix = "aws.sqs.queue"
+
+// AddMetadata adds metadata for SQS queues from a specific region
+func AddMetadata(endpoint string, regionName string, awsConfig awssdk.Config, events map[string]mb.Event) map[string]mb.Event {
+ svc := sqs.New(awscommon.EnrichAWSConfigWithEndpoint(
+ endpoint, "sqs", regionName, awsConfig))
+
+ // Get queueUrls for each region
+ queueURLs, err := getQueueUrls(svc)
+ if err != nil {
+ logp.Error(fmt.Errorf("getQueueUrls failed, skipping region %s: %w", regionName, err))
+ return events
+ }
+
+ // collect monitoring state for each instance
+ for _, queueURL := range queueURLs {
+ queueURLParsed := strings.Split(queueURL, "/")
+ queueName := queueURLParsed[len(queueURLParsed)-1]
+ if _, ok := events[queueName]; !ok {
+ continue
+ }
+ events[queueName].RootFields.Put(metadataPrefix+"name", queueName)
+ }
+ return events
+}
+
+func getQueueUrls(svc sqsiface.ClientAPI) ([]string, error) {
+ // ListQueues
+ listQueuesInput := &sqs.ListQueuesInput{}
+ req := svc.ListQueuesRequest(listQueuesInput)
+ output, err := req.Send(context.TODO())
+ if err != nil {
+ err = errors.Wrap(err, "Error ListQueues")
+ return nil, err
+ }
+ return output.QueueUrls, nil
+}

x-pack/metricbeat/module/aws/module.yml

@@ -3,6 +3,7 @@ metricsets:
  - ec2
  - elb
  - ebs
+ - sqs
  - usage
  - sns
  - s3_daily_storage

x-pack/metricbeat/module/aws/sqs/_meta/data.json

@@ -1,32 +1,41 @@
 {
  "@timestamp": "2017-10-12T08:05:34.853Z",
  "aws": {
+ "cloudwatch": {
+ "namespace": "AWS/SQS"
+ },
+ "dimensions": {
+ "QueueName": "filebeat-aws-elb-test"
+ },
  "sqs": {
  "empty_receives": 0,
  "messages": {
  "delayed": 0,
  "deleted": 0,
  "not_visible": 0,
  "received": 0,
- "sent": 0,
- "visible": 2
+ "sent": 0.2857142857142857,
+ "visible": 827.8
  },
  "oldest_message_age": {
- "sec": 78494
+ "sec": 345606.4
  },
- "queue": {
- "name": "test-s3-notification"
- },
- "sent_message_size": {}
+ "queuename": "filebeat-aws-elb-test",
+ "sent_message_size": {
+ "bytes": 1006.5
+ }
+ },
+ "tags": {
+ "created-by": "kaiyan"
  }
  },
  "cloud": {
  "account": {
- "id": "627959692251",
- "name": "elastic-test"
+ "id": "428152502467",
+ "name": "elastic-beats"
  },
  "provider": "aws",
- "region": "ap-southeast-1"
+ "region": "eu-central-1"
  },
  "event": {
  "dataset": "aws.sqs",

x-pack/metricbeat/module/aws/sqs/manifest.yml

@@ -0,0 +1,46 @@
+default: true
+input:
+ module: aws
+ metricset: cloudwatch
+ defaults:
+ metrics:
+ - namespace: AWS/SQS
+ resource_type: sqs
+ statistic: ["Average"]
+ name:
+ - ApproximateAgeOfOldestMessage
+ - ApproximateNumberOfMessagesDelayed
+ - ApproximateNumberOfMessagesNotVisible
+ - ApproximateNumberOfMessagesVisible
+ - NumberOfMessagesDeleted
+ - NumberOfMessagesReceived
+ - NumberOfMessagesSent
+ - NumberOfEmptyReceives
+ - SentMessageSize
+processors:
+ - rename:
+ ignore_missing: true
+ fields:
+ - from: "aws.sqs.metrics.ApproximateAgeOfOldestMessage.avg"
+ to: "aws.sqs.oldest_message_age.sec"
+ - from: "aws.sqs.metrics.ApproximateNumberOfMessagesDelayed.avg"
+ to: "aws.sqs.messages.delayed"
+ - from: "aws.sqs.metrics.ApproximateNumberOfMessagesNotVisible.avg"
+ to: "aws.sqs.messages.not_visible"
+ - from: "aws.sqs.metrics.ApproximateNumberOfMessagesVisible.avg"
+ to: "aws.sqs.messages.visible"
+ - from: "aws.sqs.metrics.NumberOfMessagesDeleted.avg"
+ to: "aws.sqs.messages.deleted"
+ - from: "aws.sqs.metrics.NumberOfMessagesReceived.avg"
+ to: "aws.sqs.messages.received"
+ - from: "aws.sqs.metrics.NumberOfMessagesSent.avg"
+ to: "aws.sqs.messages.sent"
+ - from: "aws.sqs.metrics.NumberOfEmptyReceives.avg"
+ to: "aws.sqs.empty_receives"
+ - from: "aws.sqs.metrics.SentMessageSize.avg"
+ to: "aws.sqs.sent_message_size.bytes"
+
+ - drop_fields:
+ ignore_missing: true
+ fields:
+ - "aws.sqs.metrics"