From 22ddcfe97b507e8334e85b611053717e78cdaad1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Thu, 17 Feb 2022 14:47:39 +0100 Subject: [PATCH] Set `log.offset` to the start of the reported line in filestream (#30445) (cherry picked from commit 8aca67327eb9bc656fc899034601c89ff3839cb9) --- CHANGELOG.next.asciidoc | 1 + libbeat/reader/readfile/metafields.go | 6 ++++-- libbeat/reader/readfile/metafields_test.go | 3 ++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 54cc67a36f5..9bca2c5b1fd 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -46,6 +46,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix using log_group_name_prefix in aws-cloudwatch input. {pull}29695[29695] - Fix multiple instances of the same module configured within `filebeat.modules` in filebeat.yml. {issue}29649[29649] {pull}29952[29952] - aws-s3: fix race condition in states used by s3-poller. {issue}30123[30123] {pull}30131[30131] +- Report the starting offset of the line in `log.offset` when using `filestream` instead of the end to be ECS compliant. {pull}30445[30445] *Filebeat* - Fix broken Kafka input {issue}29746[29746] {pull}30277[30277] diff --git a/libbeat/reader/readfile/metafields.go b/libbeat/reader/readfile/metafields.go index 734069b5950..1b5cb8bed20 100644 --- a/libbeat/reader/readfile/metafields.go +++ b/libbeat/reader/readfile/metafields.go @@ -41,10 +41,9 @@ func NewFilemeta(r reader.Reader, path string, offset int64) reader.Reader { func (r *FileMetaReader) Next() (reader.Message, error) { message, err := r.reader.Next() - r.offset += int64(message.Bytes) - // if the message is empty, there is no need to enrich it with file metadata if message.IsEmpty() { + r.offset += int64(message.Bytes) return message, err } @@ -56,6 +55,9 @@ func (r *FileMetaReader) Next() (reader.Message, error) { }, }, }) + + r.offset += int64(message.Bytes) + return message, err } diff --git a/libbeat/reader/readfile/metafields_test.go b/libbeat/reader/readfile/metafields_test.go index 978591c1b1b..a1480e6291b 100644 --- a/libbeat/reader/readfile/metafields_test.go +++ b/libbeat/reader/readfile/metafields_test.go @@ -54,7 +54,6 @@ func TestMetaFieldsOffset(t *testing.T) { if err == io.EOF { break } - offset += int64(msg.Bytes) expectedFields := common.MapStr{} if len(msg.Content) != 0 { @@ -67,6 +66,8 @@ func TestMetaFieldsOffset(t *testing.T) { }, } } + offset += int64(msg.Bytes) + require.Equal(t, expectedFields, msg.Fields) require.Equal(t, offset, in.offset) }