[x-pack][filebeat] incorporate LastModified into s3 event _id generation

CHANGELOG.next.asciidoc

@@ -371,6 +371,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - The environment variable `BEATS_AZURE_EVENTHUB_INPUT_TRACING_ENABLED: true` enables internal logs tracer for the azure-eventhub input. {issue}41931[41931] {pull}41932[41932]
 - Rate limiting operability improvements in the Okta provider of the Entity Analytics input. {issue}40106[40106] {pull}41977[41977]
 - Added default values in the streaming input for websocket retries and put a cap on retry wait time to be lesser than equal to the maximum defined wait time. {pull}42012[42012]
+- The `_id` generation process for S3 events has been updated to incorporate the LastModified field. This enhancement ensures that the `_id` is unique. {pull}42078[42078]
 
 *Auditbeat*
 

x-pack/filebeat/input/awss3/s3_input.go

@@ -256,5 +256,6 @@ func (in *s3PollerInput) s3EventForState(state state) s3EventV2 {
 	event.S3.Bucket.Name = state.Bucket
 	event.S3.Bucket.ARN = in.config.getBucketARN()
 	event.S3.Object.Key = state.Key
+	event.S3.Object.LastModified = state.LastModified
 	return event
 }

x-pack/filebeat/input/awss3/s3_objects.go

@@ -486,6 +486,7 @@ func s3ObjectHash(obj s3EventV2) string {
 	h := sha256.New()
 	h.Write([]byte(obj.S3.Bucket.ARN))
 	h.Write([]byte(obj.S3.Object.Key))
+	h.Write([]byte(obj.S3.Object.LastModified.String()))
 	prefix := hex.EncodeToString(h.Sum(nil))
 	return prefix[:10]
 }

x-pack/filebeat/input/awss3/sqs_s3_event.go

@@ -77,7 +77,8 @@ type s3EventV2 struct {
 			ARN  string `json:"arn"`
 		} `json:"bucket"`
 		Object struct {
-			Key string `json:"key"`
+			Key          string    `json:"key"`
+			LastModified time.Time `json:"lastModified"`
 		} `json:"object"`
 	} `json:"s3"`
 }