Change to use one period between start and end time (#27327) (#27403)

(cherry picked from commit 35454a7)

Co-authored-by: kaiyan-sheng <kaiyan.sheng@elastic.co>

CHANGELOG.next.asciidoc

@@ -234,6 +234,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix `logstash` module when `xpack.enabled: true` is set from emitting redundant events. {pull}22808[22808]
 - Change vsphere.datastore.capacity.used.pct value to betweeen 0 and 1. {pull}23148[23148]
 - Change `server_status_path` default setting to `nginx_status` for the `nginx` module. {pull}26642[26642]
+- Fix cloudwatch metricset collecting duplicate data points. {pull}27327[27327]
 - Fix cloudwatch metricset collecting duplicate data points. {pull}27248[27248]
 
 *Packetbeat*

metricbeat/docs/modules/aws.asciidoc

@@ -36,9 +36,10 @@ AWS regions. If `endpoint` is specified, `regions` becomes a required config par
 * *latency*
 
 Some AWS services send monitoring metrics to CloudWatch with a latency to
-process larger than Metricbeat collection period. This case, please specify a
-`latency` parameter so collection start time and end time will be shifted by the
-given latency amount.
+process larger than Metricbeat collection period. This will cause data points missing
+or none get collected by Metricbeat. In this case, please specify a `latency`
+parameter so collection start time and end time will be shifted by the given
+latency amount.
 
 * *endpoint*
 

x-pack/metricbeat/module/aws/_meta/docs.asciidoc

@@ -28,9 +28,10 @@ AWS regions. If `endpoint` is specified, `regions` becomes a required config par
 * *latency*
 
 Some AWS services send monitoring metrics to CloudWatch with a latency to
-process larger than Metricbeat collection period. This case, please specify a
-`latency` parameter so collection start time and end time will be shifted by the
-given latency amount.
+process larger than Metricbeat collection period. This will cause data points missing
+or none get collected by Metricbeat. In this case, please specify a `latency`
+parameter so collection start time and end time will be shifted by the given
+latency amount.
 
 * *endpoint*
 

x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go

@@ -1621,5 +1621,5 @@ func TestGetStartTimeEndTime(t *testing.T) {
 	m.MetricSet = &aws.MetricSet{Period: 5 * time.Minute}
 	m.logger = logp.NewLogger("test")
 	startTime, endTime := aws.GetStartTimeEndTime(m.MetricSet.Period, m.MetricSet.Latency)
-	assert.Equal(t, 9*time.Minute+59*time.Second, endTime.Sub(startTime))
+	assert.Equal(t, 5*time.Minute, endTime.Sub(startTime))
 }

x-pack/metricbeat/module/aws/utils.go

@@ -16,9 +16,6 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi"
 	"github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi/resourcegroupstaggingapiiface"
 	"github.com/pkg/errors"
-
-	"github.com/elastic/beats/v7/libbeat/common"
-	s "github.com/elastic/beats/v7/libbeat/common/schema"
 )
 
 // GetStartTimeEndTime function uses durationString to create startTime and endTime for queries.
@@ -29,11 +26,14 @@ func GetStartTimeEndTime(period time.Duration, latency time.Duration) (time.Time
 		endTime = endTime.Add(latency * -1)
 	}
 
-	// Set startTime double the period plus one second earlier than the endTime in order to
-	// make sure GetMetricDataRequest gets the latest data point for each metric. The plus
-	// one second is to make sure the startTime of the next collect period is one second later
-	// than the endTime of last collection period. This is to avoid collecting duplicate data.
-	return endTime.Add(period*-2 + time.Second), endTime
+	// Set startTime to be one period earlier than the endTime. If metrics are
+	// not being collected, use latency config parameter to offset the startTime
+	// and endTime.
+	startTime := endTime.Add(period * -1)
+	// Defining duration
+	d := 60 * time.Second
+	// Calling Round() method
+	return startTime.Round(d), endTime.Round(d)
 }
 
 // GetListMetricsOutput function gets listMetrics results from cloudwatch per namespace for each region.
@@ -98,11 +98,6 @@ func GetMetricDataResults(metricDataQueries []cloudwatch.MetricDataQuery, svc cl
 	return getMetricDataOutput.MetricDataResults, nil
 }
 
-// EventMapping maps data in input to a predefined schema.
-func EventMapping(input map[string]interface{}, schema s.Schema) (common.MapStr, error) {
-	return schema.Apply(input, s.FailOnRequired)
-}
-
 // CheckTimestampInArray checks if input timestamp exists in timestampArray and if it exists, return the position.
 func CheckTimestampInArray(timestamp time.Time, timestampArray []time.Time) (bool, int) {
 	for i := 0; i < len(timestampArray); i++ {