Skip to content

Commit

Permalink
expression: Improve the compatibility of str_to_date (#25386) (#25767)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ti-srebot
ti-srebot committed Jun 28, 2021
1 parent 64b7ecd commit c8b2f25
Show file tree
Hide file tree
Showing 3 changed files with 238 additions and 95 deletions.
70 changes: 44 additions & 26 deletions expression/builtin_time_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1397,41 +1397,52 @@ func (s *testEvaluatorSuite) TestUTCDate(c *C) {
}

func (s *testEvaluatorSuite) TestStrToDate(c *C) {
// If you want to add test cases for `strToDate` but not the builtin function,
// adding cases in `types.format_test.go` `TestStrToDate` maybe more clear and easier
tests := []struct {
Date string
Format string
Success bool
Kind byte
Expect time.Time
}{
{"10/28/2011 9:46:29 pm", "%m/%d/%Y %l:%i:%s %p", true, time.Date(2011, 10, 28, 21, 46, 29, 0, time.Local)},
{"10/28/2011 9:46:29 Pm", "%m/%d/%Y %l:%i:%s %p", true, time.Date(2011, 10, 28, 21, 46, 29, 0, time.Local)},
{"2011/10/28 9:46:29 am", "%Y/%m/%d %l:%i:%s %p", true, time.Date(2011, 10, 28, 9, 46, 29, 0, time.Local)},
{"20161122165022", `%Y%m%d%H%i%s`, true, time.Date(2016, 11, 22, 16, 50, 22, 0, time.Local)},
{"2016 11 22 16 50 22", `%Y%m%d%H%i%s`, true, time.Date(2016, 11, 22, 16, 50, 22, 0, time.Local)},
{"16-50-22 2016 11 22", `%H-%i-%s%Y%m%d`, true, time.Date(2016, 11, 22, 16, 50, 22, 0, time.Local)},
{"16-50 2016 11 22", `%H-%i-%s%Y%m%d`, false, time.Time{}},
{"15-01-2001 1:59:58.999", "%d-%m-%Y %I:%i:%s.%f", true, time.Date(2001, 1, 15, 1, 59, 58, 999000000, time.Local)},
{"15-01-2001 1:59:58.1", "%d-%m-%Y %H:%i:%s.%f", true, time.Date(2001, 1, 15, 1, 59, 58, 100000000, time.Local)},
{"15-01-2001 1:59:58.", "%d-%m-%Y %H:%i:%s.%f", true, time.Date(2001, 1, 15, 1, 59, 58, 000000000, time.Local)},
{"15-01-2001 1:9:8.999", "%d-%m-%Y %H:%i:%s.%f", true, time.Date(2001, 1, 15, 1, 9, 8, 999000000, time.Local)},
{"15-01-2001 1:9:8.999", "%d-%m-%Y %H:%i:%S.%f", true, time.Date(2001, 1, 15, 1, 9, 8, 999000000, time.Local)},
{"2003-01-02 10:11:12 PM", "%Y-%m-%d %H:%i:%S %p", false, time.Time{}},
{"10:20:10AM", "%H:%i:%S%p", false, time.Time{}},
{"10/28/2011 9:46:29 pm", "%m/%d/%Y %l:%i:%s %p", true, types.KindMysqlTime, time.Date(2011, 10, 28, 21, 46, 29, 0, time.Local)},
{"10/28/2011 9:46:29 Pm", "%m/%d/%Y %l:%i:%s %p", true, types.KindMysqlTime, time.Date(2011, 10, 28, 21, 46, 29, 0, time.Local)},
{"2011/10/28 9:46:29 am", "%Y/%m/%d %l:%i:%s %p", true, types.KindMysqlTime, time.Date(2011, 10, 28, 9, 46, 29, 0, time.Local)},
{"20161122165022", `%Y%m%d%H%i%s`, true, types.KindMysqlTime, time.Date(2016, 11, 22, 16, 50, 22, 0, time.Local)},
{"2016 11 22 16 50 22", `%Y%m%d%H%i%s`, true, types.KindMysqlTime, time.Date(2016, 11, 22, 16, 50, 22, 0, time.Local)},
{"16-50-22 2016 11 22", `%H-%i-%s%Y%m%d`, true, types.KindMysqlTime, time.Date(2016, 11, 22, 16, 50, 22, 0, time.Local)},
{"16-50 2016 11 22", `%H-%i-%s%Y%m%d`, false, types.KindMysqlTime, time.Time{}},
{"15-01-2001 1:59:58.999", "%d-%m-%Y %I:%i:%s.%f", true, types.KindMysqlTime, time.Date(2001, 1, 15, 1, 59, 58, 999000000, time.Local)},
{"15-01-2001 1:59:58.1", "%d-%m-%Y %H:%i:%s.%f", true, types.KindMysqlTime, time.Date(2001, 1, 15, 1, 59, 58, 100000000, time.Local)},
{"15-01-2001 1:59:58.", "%d-%m-%Y %H:%i:%s.%f", true, types.KindMysqlTime, time.Date(2001, 1, 15, 1, 59, 58, 000000000, time.Local)},
{"15-01-2001 1:9:8.999", "%d-%m-%Y %H:%i:%s.%f", true, types.KindMysqlTime, time.Date(2001, 1, 15, 1, 9, 8, 999000000, time.Local)},
{"15-01-2001 1:9:8.999", "%d-%m-%Y %H:%i:%S.%f", true, types.KindMysqlTime, time.Date(2001, 1, 15, 1, 9, 8, 999000000, time.Local)},
{"2003-01-02 10:11:12 PM", "%Y-%m-%d %H:%i:%S %p", false, types.KindMysqlTime, time.Time{}},
{"10:20:10AM", "%H:%i:%S%p", false, types.KindMysqlTime, time.Time{}},
// test %@(skip alpha), %#(skip number), %.(skip punct)
{"2020-10-10ABCD", "%Y-%m-%d%@", true, time.Date(2020, 10, 10, 0, 0, 0, 0, time.Local)},
{"2020-10-101234", "%Y-%m-%d%#", true, time.Date(2020, 10, 10, 0, 0, 0, 0, time.Local)},
{"2020-10-10....", "%Y-%m-%d%.", true, time.Date(2020, 10, 10, 0, 0, 0, 0, time.Local)},
{"2020-10-10.1", "%Y-%m-%d%.%#%@", true, time.Date(2020, 10, 10, 0, 0, 0, 0, time.Local)},
{"abcd2020-10-10.1", "%@%Y-%m-%d%.%#%@", true, time.Date(2020, 10, 10, 0, 0, 0, 0, time.Local)},
{"abcd-2020-10-10.1", "%@-%Y-%m-%d%.%#%@", true, time.Date(2020, 10, 10, 0, 0, 0, 0, time.Local)},
{"2020-10-10", "%Y-%m-%d%@", true, time.Date(2020, 10, 10, 0, 0, 0, 0, time.Local)},
{"2020-10-10abcde123abcdef", "%Y-%m-%d%@%#", true, time.Date(2020, 10, 10, 0, 0, 0, 0, time.Local)},
{"2020-10-10ABCD", "%Y-%m-%d%@", true, types.KindMysqlTime, time.Date(2020, 10, 10, 0, 0, 0, 0, time.Local)},
{"2020-10-101234", "%Y-%m-%d%#", true, types.KindMysqlTime, time.Date(2020, 10, 10, 0, 0, 0, 0, time.Local)},
{"2020-10-10....", "%Y-%m-%d%.", true, types.KindMysqlTime, time.Date(2020, 10, 10, 0, 0, 0, 0, time.Local)},
{"2020-10-10.1", "%Y-%m-%d%.%#%@", true, types.KindMysqlTime, time.Date(2020, 10, 10, 0, 0, 0, 0, time.Local)},
{"abcd2020-10-10.1", "%@%Y-%m-%d%.%#%@", true, types.KindMysqlTime, time.Date(2020, 10, 10, 0, 0, 0, 0, time.Local)},
{"abcd-2020-10-10.1", "%@-%Y-%m-%d%.%#%@", true, types.KindMysqlTime, time.Date(2020, 10, 10, 0, 0, 0, 0, time.Local)},
{"2020-10-10", "%Y-%m-%d%@", true, types.KindMysqlTime, time.Date(2020, 10, 10, 0, 0, 0, 0, time.Local)},
{"2020-10-10abcde123abcdef", "%Y-%m-%d%@%#", true, types.KindMysqlTime, time.Date(2020, 10, 10, 0, 0, 0, 0, time.Local)},
// some input for '%r'
{"12:3:56pm 13/05/2019", "%r %d/%c/%Y", true, types.KindMysqlTime, time.Date(2019, 5, 13, 12, 3, 56, 0, time.Local)},
{"11:13:56 am", "%r", true, types.KindMysqlDuration, time.Date(0, 0, 0, 11, 13, 56, 0, time.Local)},
// some input for '%T'
{"12:13:56 13/05/2019", "%T %d/%c/%Y", true, types.KindMysqlTime, time.Date(2019, 5, 13, 12, 13, 56, 0, time.Local)},
{"19:3:56 13/05/2019", "%T %d/%c/%Y", true, types.KindMysqlTime, time.Date(2019, 5, 13, 19, 3, 56, 0, time.Local)},
{"21:13:24", "%T", true, types.KindMysqlDuration, time.Date(0, 0, 0, 21, 13, 24, 0, time.Local)},
}

fc := funcs[ast.StrToDate]
for _, test := range tests {
date := types.NewStringDatum(test.Date)
format := types.NewStringDatum(test.Format)
c.Logf("input: %s, format: %s", test.Date, test.Format)
f, err := fc.getFunction(s.ctx, s.datumsToConstants([]types.Datum{date, format}))
c.Assert(err, IsNil)
result, err := evalBuiltinFunc(f, chunk.Row{})
Expand All @@ -1441,10 +1452,17 @@ func (s *testEvaluatorSuite) TestStrToDate(c *C) {
c.Assert(result.IsNull(), IsTrue)
continue
}
c.Assert(result.Kind(), Equals, types.KindMysqlTime)
value := result.GetMysqlTime()
t1, _ := value.GoTime(time.Local)
c.Assert(t1, Equals, test.Expect)
c.Assert(result.Kind(), Equals, test.Kind)
switch test.Kind {
case types.KindMysqlTime:
value := result.GetMysqlTime()
t1, _ := value.GoTime(time.Local)
c.Assert(t1, Equals, test.Expect)
case types.KindMysqlDuration:
value := result.GetMysqlDuration()
timeExpect := test.Expect.Sub(time.Date(0, 0, 0, 0, 0, 0, 0, time.Local))
c.Assert(value.Duration, Equals, timeExpect)
}
}
}

Expand Down
54 changes: 49 additions & 5 deletions types/format_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,30 +118,74 @@ func (s *testTimeSuite) TestStrToDate(c *C) {
{`70/10/22`, `%Y/%m/%d`, types.FromDate(1970, 10, 22, 0, 0, 0, 0)},
{`18/10/22`, `%Y/%m/%d`, types.FromDate(2018, 10, 22, 0, 0, 0, 0)},
{`100/10/22`, `%Y/%m/%d`, types.FromDate(100, 10, 22, 0, 0, 0, 0)},
//'%b'/'%M' should be case insensitive
{"31/may/2016 12:34:56.1234", "%d/%b/%Y %H:%i:%S.%f", types.FromDate(2016, 5, 31, 12, 34, 56, 123400)},
{"30/april/2016 12:34:56.", "%d/%M/%Y %H:%i:%s.%f", types.FromDate(2016, 4, 30, 12, 34, 56, 0)},
{"31/mAy/2016 12:34:56.1234", "%d/%b/%Y %H:%i:%S.%f", types.FromDate(2016, 5, 31, 12, 34, 56, 123400)},
{"30/apRil/2016 12:34:56.", "%d/%M/%Y %H:%i:%s.%f", types.FromDate(2016, 4, 30, 12, 34, 56, 0)},
// '%r'
{" 04 :13:56 AM13/05/2019", "%r %d/%c/%Y", types.FromDate(2019, 5, 13, 4, 13, 56, 0)}, //
{"12: 13:56 AM 13/05/2019", "%r%d/%c/%Y", types.FromDate(2019, 5, 13, 0, 13, 56, 0)}, //
{"12:13 :56 pm 13/05/2019", "%r %d/%c/%Y", types.FromDate(2019, 5, 13, 12, 13, 56, 0)}, //
{"12:3: 56pm 13/05/2019", "%r %d/%c/%Y", types.FromDate(2019, 5, 13, 12, 3, 56, 0)}, //
{"11:13:56", "%r", types.FromDate(0, 0, 0, 11, 13, 56, 0)}, // EOF before parsing "AM"/"PM"
{"11:13", "%r", types.FromDate(0, 0, 0, 11, 13, 0, 0)}, // EOF after hh:mm
{"11:", "%r", types.FromDate(0, 0, 0, 11, 0, 0, 0)}, // EOF after hh:
{"11", "%r", types.FromDate(0, 0, 0, 11, 0, 0, 0)}, // EOF after hh:
{"12", "%r", types.FromDate(0, 0, 0, 0, 0, 0, 0)}, // EOF after hh:, and hh=12 -> 0
// '%T'
{" 4 :13:56 13/05/2019", "%T %d/%c/%Y", types.FromDate(2019, 5, 13, 4, 13, 56, 0)},
{"23: 13:56 13/05/2019", "%T%d/%c/%Y", types.FromDate(2019, 5, 13, 23, 13, 56, 0)},
{"12:13 :56 13/05/2019", "%T %d/%c/%Y", types.FromDate(2019, 5, 13, 12, 13, 56, 0)},
{"19:3: 56 13/05/2019", "%T %d/%c/%Y", types.FromDate(2019, 5, 13, 19, 3, 56, 0)},
{"21:13", "%T", types.FromDate(0, 0, 0, 21, 13, 0, 0)}, // EOF after hh:mm
{"21:", "%T", types.FromDate(0, 0, 0, 21, 0, 0, 0)}, // EOF after hh:
// More patterns than input string
{" 2/Jun", "%d/%b/%Y", types.FromDate(0, 6, 2, 0, 0, 0, 0)},
{" liter", "lit era l", types.ZeroCoreTime},
// Feb 29 in leap-year
{"29/Feb/2020 12:34:56.", "%d/%b/%Y %H:%i:%s.%f", types.FromDate(2020, 2, 29, 12, 34, 56, 0)},
// When `AllowInvalidDate` is true, check only that the month is in the range from 1 to 12 and the day is in the range from 1 to 31
{"31/April/2016 12:34:56.", "%d/%M/%Y %H:%i:%s.%f", types.FromDate(2016, 4, 31, 12, 34, 56, 0)}, // April 31th
{"29/Feb/2021 12:34:56.", "%d/%b/%Y %H:%i:%s.%f", types.FromDate(2021, 2, 29, 12, 34, 56, 0)}, // Feb 29 in non-leap-year
{"30/Feb/2016 12:34:56.1234", "%d/%b/%Y %H:%i:%S.%f", types.FromDate(2016, 2, 30, 12, 34, 56, 123400)}, // Feb 30th
}
for i, tt := range tests {
sc.AllowInvalidDate = true
var t types.Time
c.Assert(t.StrToDate(sc, tt.input, tt.format), IsTrue, Commentf("no.%d failed", i))
c.Assert(t.CoreTime(), Equals, tt.expect, Commentf("no.%d failed", i))
c.Assert(t.StrToDate(sc, tt.input, tt.format), IsTrue, Commentf("no.%d failed input=%s format=%s", i, tt.input, tt.format))
c.Assert(t.CoreTime(), Equals, tt.expect, Commentf("no.%d failed input=%s format=%s", i, tt.input, tt.format))
}

errTests := []struct {
input string
format string
}{
{`04/31/2004`, `%m/%d/%Y`},
// invalid days when `AllowInvalidDate` is false
{`04/31/2004`, `%m/%d/%Y`}, // not exists in the real world
{"29/Feb/2021 12:34:56.", "%d/%b/%Y %H:%i:%s.%f"}, // Feb 29 in non-leap-year

{`a09:30:17`, `%h:%i:%s`}, // format mismatch
{`12:43:24`, `%r`}, // no PM or AM followed
{`12:43:24 a`, `%r`}, // followed by incomplete 'AM'/'PM'
{`23:60:12`, `%T`}, // invalid minute
{`18`, `%l`},
{`00:21:22 AM`, `%h:%i:%s %p`},
{`100/10/22`, `%y/%m/%d`},
{"2010-11-12 11 am", `%Y-%m-%d %H %p`},
{"2010-11-12 13 am", `%Y-%m-%d %h %p`},
{"2010-11-12 0 am", `%Y-%m-%d %h %p`},
// MySQL accept `SEPTEMB` as `SEPTEMBER`, but we don't want this "feature" in TiDB
// unless we have to.
{"15 SEPTEMB 2001", "%d %M %Y"},
// '%r'
{"13:13:56 AM13/5/2019", "%r"}, // hh = 13 with am is invalid
{"00:13:56 AM13/05/2019", "%r"}, // hh = 0 with am is invalid
{"00:13:56 pM13/05/2019", "%r"}, // hh = 0 with pm is invalid
{"11:13:56a", "%r"}, // EOF while parsing "AM"/"PM"
}
for i, tt := range errTests {
sc.AllowInvalidDate = false
var t types.Time
c.Assert(t.StrToDate(sc, tt.input, tt.format), IsFalse, Commentf("no.%d failed", i))
c.Assert(t.StrToDate(sc, tt.input, tt.format), IsFalse, Commentf("no.%d failed input=%s format=%s", i, tt.input, tt.format))
}
}
Loading

0 comments on commit c8b2f25

Please sign in to comment.