From 11aa1802aad2cd8745c09515636b885aee0086e2 Mon Sep 17 00:00:00 2001 From: Felix Ruess Date: Mon, 6 Jul 2020 16:00:26 +0200 Subject: [PATCH 1/5] support unix timestamps with fractional seconds see #2193 --- pkg/logentry/stages/util.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pkg/logentry/stages/util.go b/pkg/logentry/stages/util.go index bd4eac044fdbc..b94daaaf2734c 100644 --- a/pkg/logentry/stages/util.go +++ b/pkg/logentry/stages/util.go @@ -2,6 +2,7 @@ package stages import ( "fmt" + "math" "strconv" "strings" "time" @@ -64,6 +65,19 @@ func convertDateLayout(predef string, location *time.Location) parser { } case "Unix": return func(t string) (time.Time, error) { + if strings.Count(t, ".") == 1 { + split := strings.Split(t, ".") + sec, err := strconv.ParseInt(split[0], 10, 64) + if err != nil { + return time.Time{}, err + } + frac, err := strconv.ParseInt(split[1], 10, 64) + if err != nil { + return time.Time{}, err + } + nsec := int64(float64(frac) * math.Pow(10, float64(9-len(split[1])))) + return time.Unix(sec, nsec), nil + } i, err := strconv.ParseInt(t, 10, 64) if err != nil { return time.Time{}, err From 9c26e089110319d89aa905e1251958a696c744af Mon Sep 17 00:00:00 2001 From: Felix Ruess Date: Mon, 6 Jul 2020 16:25:54 +0200 Subject: [PATCH 2/5] add test for parsing unix timestamp with fractions --- pkg/logentry/stages/timestamp_test.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/pkg/logentry/stages/timestamp_test.go b/pkg/logentry/stages/timestamp_test.go index 2736c812d0b46..b2f31d44802a4 100644 --- a/pkg/logentry/stages/timestamp_test.go +++ b/pkg/logentry/stages/timestamp_test.go @@ -223,6 +223,28 @@ func TestTimestampStage_Process(t *testing.T) { }, time.Date(2019, 7, 9, 21, 48, 36, 0, time.UTC), }, + "unix fractions ms success": { + TimestampConfig{ + Source: "ts", + Format: "Unix", + }, + map[string]interface{}{ + "somethigelse": "notimportant", + "ts": "1562708916.414123", + }, + time.Date(2019, 7, 9, 21, 48, 36, 414123*1000, time.UTC), + }, + "unix fractions ns success": { + TimestampConfig{ + Source: "ts", + Format: "Unix", + }, + map[string]interface{}{ + "somethigelse": "notimportant", + "ts": "1562708916.000000123", + }, + time.Date(2019, 7, 9, 21, 48, 36, 123, time.UTC), + }, "unix millisecond success": { TimestampConfig{ Source: "ts", From 52c3ddf3930289b3caecf16caaf5618004fb3563 Mon Sep 17 00:00:00 2001 From: Felix Ruess Date: Mon, 6 Jul 2020 16:33:14 +0200 Subject: [PATCH 3/5] mention Unix timestamp with fractions in docs --- docs/clients/promtail/stages/timestamp.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/clients/promtail/stages/timestamp.md b/docs/clients/promtail/stages/timestamp.md index 9aac07abb1b01..3c1de27a949c7 100644 --- a/docs/clients/promtail/stages/timestamp.md +++ b/docs/clients/promtail/stages/timestamp.md @@ -44,7 +44,7 @@ of pre-defined formats to represent common forms: Additionally, support for common Unix timestamps is supported with the following `format` values: -- `Unix`: `1562708916` +- `Unix`: `1562708916` or with fractions `1562708916.000000123` - `UnixMs`: `1562708916414` - `UnixUs`: `1562708916414123` - `UnixNs`: `1562708916000000123` From 399a149843a59559d5d3b6be9e37ebd50a55ac87 Mon Sep 17 00:00:00 2001 From: Felix Ruess Date: Mon, 6 Jul 2020 17:19:36 +0200 Subject: [PATCH 4/5] return error if timestamp is not split into two parts --- pkg/logentry/stages/util.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/logentry/stages/util.go b/pkg/logentry/stages/util.go index b94daaaf2734c..3c057a75b80ed 100644 --- a/pkg/logentry/stages/util.go +++ b/pkg/logentry/stages/util.go @@ -67,6 +67,9 @@ func convertDateLayout(predef string, location *time.Location) parser { return func(t string) (time.Time, error) { if strings.Count(t, ".") == 1 { split := strings.Split(t, ".") + if len(split) != 2 { + return time.Time{}, fmt.Errorf("Can't split %v into two parts", t) + } sec, err := strconv.ParseInt(split[0], 10, 64) if err != nil { return time.Time{}, err From f65e65ea2fa6aeb38ebc1cba52e8da7360c0036b Mon Sep 17 00:00:00 2001 From: Felix Ruess Date: Mon, 6 Jul 2020 19:05:04 +0200 Subject: [PATCH 5/5] don't capitalize error Co-authored-by: Cyril Tovena --- pkg/logentry/stages/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/logentry/stages/util.go b/pkg/logentry/stages/util.go index 3c057a75b80ed..407387f7c486b 100644 --- a/pkg/logentry/stages/util.go +++ b/pkg/logentry/stages/util.go @@ -68,7 +68,7 @@ func convertDateLayout(predef string, location *time.Location) parser { if strings.Count(t, ".") == 1 { split := strings.Split(t, ".") if len(split) != 2 { - return time.Time{}, fmt.Errorf("Can't split %v into two parts", t) + return time.Time{}, fmt.Errorf("can't split %v into two parts", t) } sec, err := strconv.ParseInt(split[0], 10, 64) if err != nil {