diff --git a/docs/logentry/processing-log-lines.md b/docs/logentry/processing-log-lines.md index baac228a3c8fb..501df7de0ee85 100644 --- a/docs/logentry/processing-log-lines.md +++ b/docs/logentry/processing-log-lines.md @@ -360,7 +360,7 @@ A match stage will take the provided label `selector` and determine if a group o ### timestamp -A timestamp stage will parse data from the `extracted` map and set the `time` value which will be stored by Loki. +A timestamp stage will parse data from the `extracted` map and set the `time` value which will be stored by Loki. The timestamp stage is important for having log entries in the correct order. In the absence of this stage, promtail will associate the current timestamp to the log entry. ```yaml - timestamp: @@ -394,7 +394,7 @@ UnixMs = 1562708916414 UnixNs = 1562708916000000123 ``` -Finally any custom format can be supplied, and will be passed directly in as the layout parameter in time.Parse() +Finally any custom format can be supplied, and will be passed directly in as the layout parameter in `time.Parse()`. If the custom format has no year component specified (ie. syslog's default logs), promtail will assume the current year should be used, correctly handling the edge cases around new year's eve. __Read the [time.parse](https://golang.org/pkg/time/#Parse) docs closely if passing a custom format and make sure your custom format uses the special date they specify: `Mon Jan 2 15:04:05 -0700 MST 2006`__ diff --git a/pkg/logentry/stages/timestamp_test.go b/pkg/logentry/stages/timestamp_test.go index cc85c46c9fddc..5900920138bb0 100644 --- a/pkg/logentry/stages/timestamp_test.go +++ b/pkg/logentry/stages/timestamp_test.go @@ -73,7 +73,7 @@ func TestTimestampValidation(t *testing.T) { testString: "2012-11-01T22:08:41-04:00", expectedTime: time.Date(2012, 11, 01, 22, 8, 41, 0, time.FixedZone("", -4*60*60)), }, - "custom format": { + "custom format with year": { config: &TimestampConfig{ Source: "source1", Format: "2006-01-02", @@ -82,6 +82,15 @@ func TestTimestampValidation(t *testing.T) { testString: "2009-01-01", expectedTime: time.Date(2009, 01, 01, 00, 00, 00, 0, time.UTC), }, + "custom format without year": { + config: &TimestampConfig{ + Source: "source1", + Format: "Jan 02 15:04:05", + }, + err: nil, + testString: "Jul 15 01:02:03", + expectedTime: time.Date(time.Now().Year(), 7, 15, 1, 2, 3, 0, time.UTC), + }, "unix_ms": { config: &TimestampConfig{ Source: "source1", diff --git a/pkg/logentry/stages/util.go b/pkg/logentry/stages/util.go index 95a94b6e37198..fce81e56c8eef 100644 --- a/pkg/logentry/stages/util.go +++ b/pkg/logentry/stages/util.go @@ -3,9 +3,14 @@ package stages import ( "fmt" "strconv" + "strings" "time" ) +const ( + ErrTimestampContainsYear = "timestamp '%s' is expected to not contain the year date component" +) + // convertDateLayout converts pre-defined date format layout into date format func convertDateLayout(predef string) parser { switch predef { @@ -74,12 +79,44 @@ func convertDateLayout(predef string) parser { return time.Unix(0, i), nil } default: + if !strings.Contains(predef, "2006") { + return func(t string) (time.Time, error) { + return parseTimestampWithoutYear(predef, t, time.Now()) + } + } return func(t string) (time.Time, error) { return time.Parse(predef, t) } } } +// parseTimestampWithoutYear parses the input timestamp without the year component, +// assuming the timestamp is related to a point in time close to "now", and correctly +// handling the edge cases around new year's eve +func parseTimestampWithoutYear(layout string, timestamp string, now time.Time) (time.Time, error) { + parsedTime, err := time.Parse(layout, timestamp) + if err != nil { + return parsedTime, err + } + + // Ensure the year component of the input date string has not been + // parsed for real + if parsedTime.Year() != 0 { + return parsedTime, fmt.Errorf(ErrTimestampContainsYear, timestamp) + } + + // Handle the case we're crossing the new year's eve midnight + if parsedTime.Month() == 12 && now.Month() == 1 { + parsedTime = parsedTime.AddDate(now.Year()-1, 0, 0) + } else if parsedTime.Month() == 1 && now.Month() == 12 { + parsedTime = parsedTime.AddDate(now.Year()+1, 0, 0) + } else { + parsedTime = parsedTime.AddDate(now.Year(), 0, 0) + } + + return parsedTime, nil +} + // getString will convert the input variable to a string if possible func getString(unk interface{}) (string, error) { diff --git a/pkg/logentry/stages/util_test.go b/pkg/logentry/stages/util_test.go index 06f8f46cd0644..9b878bab7c7bf 100644 --- a/pkg/logentry/stages/util_test.go +++ b/pkg/logentry/stages/util_test.go @@ -1,6 +1,7 @@ package stages import ( + "fmt" "testing" "time" @@ -65,3 +66,98 @@ func TestGetString(t *testing.T) { assert.Equal(t, "2.02", s32) assert.Equal(t, "1562723913000", s64_1) } + +func TestConvertDateLayout(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + layout string + timestamp string + expected time.Time + }{ + "custom layout with year": { + "2006 Jan 02 15:04:05", + "2019 Jul 15 01:02:03", + time.Date(2019, 7, 15, 1, 2, 3, 0, time.UTC), + }, + "custom layout without year": { + "Jan 02 15:04:05", + "Jul 15 01:02:03", + time.Date(time.Now().Year(), 7, 15, 1, 2, 3, 0, time.UTC), + }, + } + + for testName, testData := range tests { + testData := testData + + t.Run(testName, func(t *testing.T) { + t.Parallel() + + parser := convertDateLayout(testData.layout) + parsed, err := parser(testData.timestamp) + if err != nil { + t.Errorf("convertDateLayout() parser returned an unexpected error = %v", err) + return + } + + assert.Equal(t, testData.expected, parsed) + }) + } +} + +func TestParseTimestampWithoutYear(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + layout string + timestamp string + now time.Time + expected time.Time + err error + }{ + "parse timestamp within current year": { + "Jan 02 15:04:05", + "Jul 15 01:02:03", + time.Date(2019, 7, 14, 0, 0, 0, 0, time.UTC), + time.Date(2019, 7, 15, 1, 2, 3, 0, time.UTC), + nil, + }, + "parse timestamp on 31th Dec and today is 1st Jan": { + "Jan 02 15:04:05", + "Dec 31 23:59:59", + time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC), + time.Date(2018, 12, 31, 23, 59, 59, 0, time.UTC), + nil, + }, + "parse timestamp on 1st Jan and today is 31st Dec": { + "Jan 02 15:04:05", + "Jan 01 01:02:03", + time.Date(2018, 12, 31, 23, 59, 59, 0, time.UTC), + time.Date(2019, 1, 1, 1, 2, 3, 0, time.UTC), + nil, + }, + "error if the input layout actually includes the year component": { + "2006 Jan 02 15:04:05", + "2019 Jan 01 01:02:03", + time.Date(2019, 1, 1, 1, 2, 3, 0, time.UTC), + time.Date(2019, 1, 1, 1, 2, 3, 0, time.UTC), + fmt.Errorf(ErrTimestampContainsYear, "2019 Jan 01 01:02:03"), + }, + } + + for testName, testData := range tests { + testData := testData + + t.Run(testName, func(t *testing.T) { + t.Parallel() + + parsed, err := parseTimestampWithoutYear(testData.layout, testData.timestamp, testData.now) + if ((err != nil) != (testData.err != nil)) || (err != nil && testData.err != nil && err.Error() != testData.err.Error()) { + t.Errorf("parseTimestampWithoutYear() expected error = %v, actual error = %v", testData.err, err) + return + } + + assert.Equal(t, testData.expected, parsed) + }) + } +}