Add fallback_formats for timestamp stage

Fixes grafana#2590
aminjam · Sep 25, 2020 · d05a4ab · d05a4ab
1 parent ee67b9c
commit d05a4ab
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 4 deletions.
diff --git a/docs/sources/clients/promtail/stages/timestamp.md b/docs/sources/clients/promtail/stages/timestamp.md
@@ -20,6 +20,12 @@ timestamp:
   # UnixMs UnixUs UnixNs].
   format: <string>
 
+  # Fallback formats to try if the format fails to parse the value
+  # Can use pre-defined formats by name: [ANSIC UnixDate RubyDate RFC822
+  # RFC822Z RFC850 RFC1123 RFC1123Z RFC3339 RFC3339Nano Unix
+  # UnixMs UnixUs UnixNs].
+  [fallback_formats: []<string>]
+
   # IANA Timezone Database string.
   [location: <string>]
 

diff --git a/pkg/logentry/stages/timestamp.go b/pkg/logentry/stages/timestamp.go
@@ -44,10 +44,11 @@ var (
 
 // TimestampConfig configures timestamp extraction
 type TimestampConfig struct {
-	Source          string  `mapstructure:"source"`
-	Format          string  `mapstructure:"format"`
-	Location        *string `mapstructure:"location"`
-	ActionOnFailure *string `mapstructure:"action_on_failure"`
+	Source          string   `mapstructure:"source"`
+	Format          string   `mapstructure:"format"`
+	FallbackFormats []string `mapstructure:"fallback_formats"`
+	Location        *string  `mapstructure:"location"`
+	ActionOnFailure *string  `mapstructure:"action_on_failure"`
 }
 
 // parser can convert the time string into a time.Time value
@@ -82,6 +83,22 @@ func validateTimestampConfig(cfg *TimestampConfig) (parser, error) {
 		}
 	}
 
+	if len(cfg.FallbackFormats) > 0 {
+		multiConvertDateLayout := func(input string) (time.Time, error) {
+			orignalTime, originalErr := convertDateLayout(cfg.Format, loc)(input)
+			if originalErr == nil {
+				return orignalTime, originalErr
+			}
+			for i := 0; i < len(cfg.FallbackFormats); i++ {
+				if t, err := convertDateLayout(cfg.FallbackFormats[i], loc)(input); err == nil {
+					return t, err
+				}
+			}
+			return orignalTime, originalErr
+		}
+		return multiConvertDateLayout, nil
+	}
+
 	return convertDateLayout(cfg.Format, loc), nil
 }
 

diff --git a/pkg/logentry/stages/timestamp_test.go b/pkg/logentry/stages/timestamp_test.go
@@ -169,6 +169,16 @@ func TestTimestampValidation(t *testing.T) {
 			},
 			err: fmt.Errorf(ErrInvalidActionOnFailure, TimestampActionOnFailureOptions),
 		},
+		"fallback formats contains the format": {
+			config: &TimestampConfig{
+				Source:          "source1",
+				Format:          "UnixMs",
+				FallbackFormats: []string{"2006-01-02 03:04:05.000000000 +0000 UTC", time.RFC3339},
+			},
+			err:          nil,
+			testString:   "2012-11-01T22:08:41-04:00",
+			expectedTime: time.Date(2012, 11, 01, 22, 8, 41, 0, time.FixedZone("", -4*60*60)),
+		},
 	}
 	for name, test := range tests {
 		test := test