From 4731cad5c518bacf9d957cfb39a47a8a2d8aa694 Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Sat, 19 Dec 2020 22:52:49 +0800 Subject: [PATCH 1/6] types: support %X %V %W formater for STR_TO_DATE() --- types/format_test.go | 1 + types/time.go | 73 ++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 68 insertions(+), 6 deletions(-) diff --git a/types/format_test.go b/types/format_test.go index 7dac7bd45105f..a930c5596f983 100644 --- a/types/format_test.go +++ b/types/format_test.go @@ -118,6 +118,7 @@ func (s *testTimeSuite) TestStrToDate(c *C) { {`70/10/22`, `%Y/%m/%d`, types.FromDate(1970, 10, 22, 0, 0, 0, 0)}, {`18/10/22`, `%Y/%m/%d`, types.FromDate(2018, 10, 22, 0, 0, 0, 0)}, {`100/10/22`, `%Y/%m/%d`, types.FromDate(100, 10, 22, 0, 0, 0, 0)}, + {`200442 Monday`, `%X%V %W`, types.FromDate(2004, 10, 18, 0, 0, 0, 0)}, } for i, tt := range tests { var t types.Time diff --git a/types/time.go b/types/time.go index 485b49734d3af..416cdb64b2089 100644 --- a/types/time.go +++ b/types/time.go @@ -2766,9 +2766,41 @@ func mysqlTimeFix(t *CoreTime, ctx map[string]int) error { if valueAMorPm == constForPM { t.setHour(t.getHour() + 12) } - } else { - if _, ok := ctx["%h"]; ok && t.Hour() == 12 { - t.setHour(0) + } else if _, ok := ctx["%h"]; ok && t.Hour() == 12 { + t.setHour(0) + } else if weekDay, ok := ctx["%W"]; ok { + if weekNumber, ok := ctx["%V"]; ok { + sundayFirstDayOfWeek := true + // Number of days since year 0 till 1st Jan of this year. + days := calcDaynr(t.Year(), 1, 1) + // Which day of week is 1st Jan of this year. + weekday1 := calcWeekday(days, sundayFirstDayOfWeek); + // Below we are going to sum: + // 1) number of days since year 0 till 1st day of 1st week of this year + // 2) number of days between 1st week and our week + // 3) and position of our day in the week + days = days + (weekNumber - 1) * 7 + if sundayFirstDayOfWeek { + if weekday1 != 0 { + days = days + 7 - weekday1 + weekDay % 7; + } else { + days = days - weekday1 + weekDay % 7; + } + } else { + if weekday1 <= 3 { + days = days - weekday1 + (weekDay - 1); + } else { + days = days + 7 - weekday1 + (weekDay - 1); + } + } + + if days <= 0 /* || days > MaxDayNumver */ { + return errors.New("....") + } + yy,mm,dd := getDateFromDaynr(uint(days)) + t.setYear(uint16(yy)) + t.setMonth(uint8(mm)) + t.setDay(uint8(dd)) } } return nil @@ -2885,11 +2917,11 @@ var dateFormatParserTable = map[string]dateFormatParser{ // "%D": dayOfMonthWithSuffix, // Day of the month with English suffix (0th, 1st, 2nd, 3rd) // "%U": weekMode0, // Week (00..53), where Sunday is the first day of the week; WEEK() mode 0 // "%u": weekMode1, // Week (00..53), where Monday is the first day of the week; WEEK() mode 1 - // "%V": weekMode2, // Week (01..53), where Sunday is the first day of the week; WEEK() mode 2; used with %X + "%V": weekMode2, // Week (01..53), where Sunday is the first day of the week; WEEK() mode 2; used with %X // "%v": weekMode3, // Week (01..53), where Monday is the first day of the week; WEEK() mode 3; used with %x - // "%W": weekdayName, // Weekday name (Sunday..Saturday) + "%W": weekdayName, // Weekday name (Sunday..Saturday) // "%w": dayOfWeek, // Day of the week (0=Sunday..6=Saturday) - // "%X": yearOfWeek, // Year for the week where Sunday is the first day of the week, numeric, four digits; used with %V + "%X": yearOfWeek, // Year for the week where Sunday is the first day of the week, numeric, four digits; used with %V // "%x": yearOfWeek, // Year for the week, where Monday is the first day of the week, numeric, four digits; used with %v } @@ -3191,6 +3223,35 @@ func fullNameMonth(t *CoreTime, input string, ctx map[string]int) (string, bool) return input, false } +func yearOfWeek(t *CoreTime, input string, ctx map[string]int) (string, bool) { + v, succ := parseDigits(input, 4) + if !succ { + return input, false + } + t.setYear(uint16(v)) + return input[4:], true +} + +func weekMode2(t *CoreTime, input string, ctx map[string]int) (string, bool) { + v, succ := parseDigits(input, 2) + // v should be in [01, 53] + if !succ || v == 0 || v > 53 { + return input, false + } + ctx["%V"] = v + return input[2:], true +} + +func weekdayName(t *CoreTime, input string, ctx map[string]int) (string, bool) { + for i, weekday := range WeekdayNames { + if strings.HasPrefix(input, weekday) { + ctx["%W"] = i + 1 + return input[len(weekday):], true + } + } + return input, false +} + func monthNumeric(t *CoreTime, input string, ctx map[string]int) (string, bool) { result := oneOrTwoDigitRegex.FindString(input) // 1..12 length := len(result) From 7adb4f722d815ab10585aba56de6c328628fa62e Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Mon, 28 Dec 2020 23:49:20 +0800 Subject: [PATCH 2/6] add test code --- types/format_test.go | 11 ++++ types/time.go | 125 ++++++++++++++++++++++++++++++++++--------- 2 files changed, 111 insertions(+), 25 deletions(-) diff --git a/types/format_test.go b/types/format_test.go index a930c5596f983..074adfc793c3c 100644 --- a/types/format_test.go +++ b/types/format_test.go @@ -84,6 +84,7 @@ func (s *testTimeSuite) TestStrToDate(c *C) { format string expect types.CoreTime }{ + {`2004420`, `%x%v%w`, types.FromDate(2004, 10, 17, 0, 0, 0, 0)}, {`01,05,2013`, `%d,%m,%Y`, types.FromDate(2013, 5, 1, 0, 0, 0, 0)}, {`May 01, 2013`, `%M %d,%Y`, types.FromDate(2013, 5, 1, 0, 0, 0, 0)}, {`a09:30:17`, `a%h:%i:%s`, types.FromDate(0, 0, 0, 9, 30, 17, 0)}, @@ -119,6 +120,11 @@ func (s *testTimeSuite) TestStrToDate(c *C) { {`18/10/22`, `%Y/%m/%d`, types.FromDate(2018, 10, 22, 0, 0, 0, 0)}, {`100/10/22`, `%Y/%m/%d`, types.FromDate(100, 10, 22, 0, 0, 0, 0)}, {`200442 Monday`, `%X%V %W`, types.FromDate(2004, 10, 18, 0, 0, 0, 0)}, + {`2004420`, `%X%V%w`, types.FromDate(2004, 10, 17, 0, 0, 0, 0)}, + {`2004423`, `%X%V%w`, types.FromDate(2004, 10, 20, 0, 0, 0, 0)}, + {`200442 Sunday`, `%x%v%W`, types.FromDate(2004, 10, 17, 0, 0, 0, 0)}, + {"2004421", "%Y%U%w", types.FromDate(2004, 10, 18, 0, 0, 0, 0)}, // %U,%u should be used with %Y and not %X or %x + {"69421", "%y%U%w", types.FromDate(2069, 10, 21, 0, 0, 0, 0)}, // %U,%u should be used with %Y and not %X or %x } for i, tt := range tests { var t types.Time @@ -140,6 +146,11 @@ func (s *testTimeSuite) TestStrToDate(c *C) { {"2010-11-12 11 am", `%Y-%m-%d %H %p`}, {"2010-11-12 13 am", `%Y-%m-%d %h %p`}, {"2010-11-12 0 am", `%Y-%m-%d %h %p`}, + {`2004427`, `%X%V%w`}, // %w range 0-6, here is 7 + {"2004421", "%x%V%w"}, // %x should be used with %v + {"2004421", "%X%v%w"}, // %X should be used with %V + {"2004421", "%X%U%w"}, // %U,%u should be used with %Y and not %X or %x + {`2004663`, `%X%V%w`}, // %V out of range [0, 53] } for i, tt := range errTests { var t types.Time diff --git a/types/time.go b/types/time.go index 416cdb64b2089..0134036f1af61 100644 --- a/types/time.go +++ b/types/time.go @@ -2768,36 +2768,49 @@ func mysqlTimeFix(t *CoreTime, ctx map[string]int) error { } } else if _, ok := ctx["%h"]; ok && t.Hour() == 12 { t.setHour(0) - } else if weekDay, ok := ctx["%W"]; ok { - if weekNumber, ok := ctx["%V"]; ok { - sundayFirstDayOfWeek := true + } else if weekNumber, ok := ctx["WeekNumber"]; ok { + weekDay := ctx["WeekDay"] + sundayFirstDayOfWeek := ctx["SundayFirst"] > 0 + if weekDay > 0 { + // %V,%v require %X,%x resprectively, + // %U,%u should be used with %Y and not %X or %x + if ctx["V"] != ctx["X"] || ctx["v"] != ctx["x"] { + return ErrWrongValue.GenWithStackByArgs(TimeStr, t) + } + if ctx["U"] > 0 || ctx["u"] > 0 { + if ctx["X"] > 0 || ctx["x"] > 0 { + return ErrWrongValue.GenWithStackByArgs(TimeStr, t) + } + } + // Number of days since year 0 till 1st Jan of this year. days := calcDaynr(t.Year(), 1, 1) // Which day of week is 1st Jan of this year. - weekday1 := calcWeekday(days, sundayFirstDayOfWeek); + weekday1 := calcWeekday(days, sundayFirstDayOfWeek) // Below we are going to sum: // 1) number of days since year 0 till 1st day of 1st week of this year // 2) number of days between 1st week and our week // 3) and position of our day in the week - days = days + (weekNumber - 1) * 7 + days = days + (weekNumber-1)*7 if sundayFirstDayOfWeek { if weekday1 != 0 { - days = days + 7 - weekday1 + weekDay % 7; + days = days + 7 - weekday1 + weekDay%7 } else { - days = days - weekday1 + weekDay % 7; + days = days - weekday1 + weekDay%7 } } else { if weekday1 <= 3 { - days = days - weekday1 + (weekDay - 1); + days = days - weekday1 + (weekDay - 1) } else { - days = days + 7 - weekday1 + (weekDay - 1); + days = days + 7 - weekday1 + (weekDay - 1) } } - if days <= 0 /* || days > MaxDayNumver */ { - return errors.New("....") + const MaxDayNumver = 3652424 + if days <= 0 || days > MaxDayNumver { + return ErrWrongValue.GenWithStackByArgs(TimeStr, t) } - yy,mm,dd := getDateFromDaynr(uint(days)) + yy, mm, dd := getDateFromDaynr(uint(days)) t.setYear(uint16(yy)) t.setMonth(uint8(mm)) t.setDay(uint8(dd)) @@ -2915,14 +2928,14 @@ var dateFormatParserTable = map[string]dateFormatParser{ // TODO: Add the following... // "%a": abbreviatedWeekday, // Abbreviated weekday name (Sun..Sat) // "%D": dayOfMonthWithSuffix, // Day of the month with English suffix (0th, 1st, 2nd, 3rd) - // "%U": weekMode0, // Week (00..53), where Sunday is the first day of the week; WEEK() mode 0 - // "%u": weekMode1, // Week (00..53), where Monday is the first day of the week; WEEK() mode 1 - "%V": weekMode2, // Week (01..53), where Sunday is the first day of the week; WEEK() mode 2; used with %X - // "%v": weekMode3, // Week (01..53), where Monday is the first day of the week; WEEK() mode 3; used with %x - "%W": weekdayName, // Weekday name (Sunday..Saturday) - // "%w": dayOfWeek, // Day of the week (0=Sunday..6=Saturday) - "%X": yearOfWeek, // Year for the week where Sunday is the first day of the week, numeric, four digits; used with %V - // "%x": yearOfWeek, // Year for the week, where Monday is the first day of the week, numeric, four digits; used with %v + "%U": weekModeU, // Week (00..53), where Sunday is the first day of the week; WEEK() mode 0 + "%u": weekModeu, // Week (00..53), where Monday is the first day of the week; WEEK() mode 1 + "%V": weekModeV, // Week (01..53), where Sunday is the first day of the week; WEEK() mode 2; used with %X + "%v": weekModev, // Week (01..53), where Monday is the first day of the week; WEEK() mode 3; used with %x + "%W": weekdayName, // Weekday name (Sunday..Saturday) + "%w": dayOfWeek, // Day of the week (0=Sunday..6=Saturday) + "%X": yearOfWeekX, // Year for the week where Sunday is the first day of the week, numeric, four digits; used with %V + "%x": yearOfWeekx, // Year for the week, where Monday is the first day of the week, numeric, four digits; used with %v } // GetFormatType checks the type(Duration, Date or Datetime) of a format string. @@ -3223,35 +3236,97 @@ func fullNameMonth(t *CoreTime, input string, ctx map[string]int) (string, bool) return input, false } -func yearOfWeek(t *CoreTime, input string, ctx map[string]int) (string, bool) { +func yearOfWeekX(t *CoreTime, input string, ctx map[string]int) (string, bool) { + v, succ := parseDigits(input, 4) + if !succ { + return input, false + } + t.setYear(uint16(v)) + ctx["X"] = 1 + return input[4:], true +} + +func yearOfWeekx(t *CoreTime, input string, ctx map[string]int) (string, bool) { v, succ := parseDigits(input, 4) if !succ { return input, false } t.setYear(uint16(v)) + ctx["x"] = 1 return input[4:], true } -func weekMode2(t *CoreTime, input string, ctx map[string]int) (string, bool) { +func weekModeV(t *CoreTime, input string, ctx map[string]int) (string, bool) { + v, succ := parseDigits(input, 2) + // v should be in [01, 53] + if !succ || v <= 0 || v > 53 { + return input, false + } + ctx["WeekNumber"] = v + ctx["SundayFirst"] = 1 + ctx["V"] = 1 + return input[2:], true +} + +func weekModeU(t *CoreTime, input string, ctx map[string]int) (string, bool) { + v, succ := parseDigits(input, 2) + // v should be in [00, 53] + if !succ || v < 0 || v > 53 { + return input, false + } + ctx["WeekNumber"] = v + ctx["SundayFirst"] = 1 + ctx["U"] = 1 + return input[2:], true +} + +func weekModev(t *CoreTime, input string, ctx map[string]int) (string, bool) { v, succ := parseDigits(input, 2) // v should be in [01, 53] - if !succ || v == 0 || v > 53 { + if !succ || v <= 0 || v > 53 { + return input, false + } + ctx["WeekNumber"] = v + ctx["v"] = 1 + return input[2:], true +} + +func weekModeu(t *CoreTime, input string, ctx map[string]int) (string, bool) { + v, succ := parseDigits(input, 2) + // v should be in [00, 53] + if !succ || v < 0 || v > 53 { return input, false } - ctx["%V"] = v + ctx["WeekNumber"] = v + ctx["u"] = 1 return input[2:], true } func weekdayName(t *CoreTime, input string, ctx map[string]int) (string, bool) { for i, weekday := range WeekdayNames { if strings.HasPrefix(input, weekday) { - ctx["%W"] = i + 1 + // Use 1-7 + ctx["WeekDay"] = i + 1 return input[len(weekday):], true } } return input, false } +func dayOfWeek(t *CoreTime, input string, ctx map[string]int) (string, bool) { + // 0=Sunday.. 6=Saturday + v, succ := parseDigits(input, 1) + if !succ || v < 0 || v > 6 { + return input, false + } + if v == 0 { + // Should use 1-7 for %w as for %W + v = 7 + } + ctx["WeekDay"] = v + return input[1:], true +} + func monthNumeric(t *CoreTime, input string, ctx map[string]int) (string, bool) { result := oneOrTwoDigitRegex.FindString(input) // 1..12 length := len(result) From ebbf199faa3099ba035fe44d67b82f05412b78a3 Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Thu, 10 Jun 2021 14:20:40 +0800 Subject: [PATCH 3/6] address comment --- types/format_test.go | 4 ++++ types/time.go | 30 ++++++++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/types/format_test.go b/types/format_test.go index 074adfc793c3c..e854c46898489 100644 --- a/types/format_test.go +++ b/types/format_test.go @@ -123,6 +123,9 @@ func (s *testTimeSuite) TestStrToDate(c *C) { {`2004420`, `%X%V%w`, types.FromDate(2004, 10, 17, 0, 0, 0, 0)}, {`2004423`, `%X%V%w`, types.FromDate(2004, 10, 20, 0, 0, 0, 0)}, {`200442 Sunday`, `%x%v%W`, types.FromDate(2004, 10, 17, 0, 0, 0, 0)}, + {`200442 Sun`, `%x%v%W`, types.FromDate(2004, 10, 17, 0, 0, 0, 0)}, + {`200442 sun`, `%x%v%W`, types.FromDate(2004, 10, 17, 0, 0, 0, 0)}, + {`200442 suNd`, `%x%v%W`, types.FromDate(2004, 10, 17, 0, 0, 0, 0)}, {"2004421", "%Y%U%w", types.FromDate(2004, 10, 18, 0, 0, 0, 0)}, // %U,%u should be used with %Y and not %X or %x {"69421", "%y%U%w", types.FromDate(2069, 10, 21, 0, 0, 0, 0)}, // %U,%u should be used with %Y and not %X or %x } @@ -151,6 +154,7 @@ func (s *testTimeSuite) TestStrToDate(c *C) { {"2004421", "%X%v%w"}, // %X should be used with %V {"2004421", "%X%U%w"}, // %U,%u should be used with %Y and not %X or %x {`2004663`, `%X%V%w`}, // %V out of range [0, 53] + {`200442 S`, `%x%v%W`}, } for i, tt := range errTests { var t types.Time diff --git a/types/time.go b/types/time.go index 75b6acf3fe66b..0cef8d58d7bd6 100644 --- a/types/time.go +++ b/types/time.go @@ -3315,15 +3315,41 @@ func weekModeu(t *CoreTime, input string, ctx map[string]int) (string, bool) { func weekdayName(t *CoreTime, input string, ctx map[string]int) (string, bool) { for i, weekday := range WeekdayNames { - if strings.HasPrefix(input, weekday) { + // The behavior is weird in MySQL, it not full string prefix match like: + // strings.HasPrefix(input, weekday) + // It's kind of a case insensitive match and success as long as the matched size > 2 + // s => false + // sn => false + // sun => true + // sUn => true + // sunda => true + // sUnday => true + matched := 0 + for matched < len(weekday) && matched < len(input) { + if !byteEqualCI(input[matched], weekday[matched]) { + break + } + matched++ + } + if matched >= 2 { // Use 1-7 ctx["WeekDay"] = i + 1 - return input[len(weekday):], true + return input[matched:], true } } return input, false } +func byteEqualCI(x, y byte) bool { + if x == y { + return true + } + if unicode.ToUpper(rune(x)) == unicode.ToUpper(rune(y)) { + return true + } + return false +} + func dayOfWeek(t *CoreTime, input string, ctx map[string]int) (string, bool) { // 0=Sunday.. 6=Saturday v, succ := parseDigits(input, 1) From 76b2cc36e281e4c3add6c9210cab1b9ddbfb7443 Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Thu, 10 Jun 2021 14:27:22 +0800 Subject: [PATCH 4/6] address comment --- types/format_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/types/format_test.go b/types/format_test.go index e854c46898489..1e95b17830455 100644 --- a/types/format_test.go +++ b/types/format_test.go @@ -125,7 +125,7 @@ func (s *testTimeSuite) TestStrToDate(c *C) { {`200442 Sunday`, `%x%v%W`, types.FromDate(2004, 10, 17, 0, 0, 0, 0)}, {`200442 Sun`, `%x%v%W`, types.FromDate(2004, 10, 17, 0, 0, 0, 0)}, {`200442 sun`, `%x%v%W`, types.FromDate(2004, 10, 17, 0, 0, 0, 0)}, - {`200442 suNd`, `%x%v%W`, types.FromDate(2004, 10, 17, 0, 0, 0, 0)}, + {`200442 suNd`, `%x%v%W`, types.FromDate(2004, 10, 17, 0, 0, 0, 0)}, // Weird MySQL behavior, matched as sunday {"2004421", "%Y%U%w", types.FromDate(2004, 10, 18, 0, 0, 0, 0)}, // %U,%u should be used with %Y and not %X or %x {"69421", "%y%U%w", types.FromDate(2069, 10, 21, 0, 0, 0, 0)}, // %U,%u should be used with %Y and not %X or %x } From 58ee901288f2cb182854711ae747c721f1ed2a87 Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Thu, 10 Jun 2021 14:33:40 +0800 Subject: [PATCH 5/6] go fmt --- types/format_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/types/format_test.go b/types/format_test.go index 1e95b17830455..de1e118007508 100644 --- a/types/format_test.go +++ b/types/format_test.go @@ -126,8 +126,8 @@ func (s *testTimeSuite) TestStrToDate(c *C) { {`200442 Sun`, `%x%v%W`, types.FromDate(2004, 10, 17, 0, 0, 0, 0)}, {`200442 sun`, `%x%v%W`, types.FromDate(2004, 10, 17, 0, 0, 0, 0)}, {`200442 suNd`, `%x%v%W`, types.FromDate(2004, 10, 17, 0, 0, 0, 0)}, // Weird MySQL behavior, matched as sunday - {"2004421", "%Y%U%w", types.FromDate(2004, 10, 18, 0, 0, 0, 0)}, // %U,%u should be used with %Y and not %X or %x - {"69421", "%y%U%w", types.FromDate(2069, 10, 21, 0, 0, 0, 0)}, // %U,%u should be used with %Y and not %X or %x + {"2004421", "%Y%U%w", types.FromDate(2004, 10, 18, 0, 0, 0, 0)}, // %U,%u should be used with %Y and not %X or %x + {"69421", "%y%U%w", types.FromDate(2069, 10, 21, 0, 0, 0, 0)}, // %U,%u should be used with %Y and not %X or %x } for i, tt := range tests { var t types.Time From 9f62bb2385b61232226acbcd81135c937a14e672 Mon Sep 17 00:00:00 2001 From: tison Date: Wed, 14 Jul 2021 10:58:13 +0800 Subject: [PATCH 6/6] gofmt Signed-off-by: tison --- types/format_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/types/format_test.go b/types/format_test.go index 34d4927e25956..0b9b0124bbd9c 100644 --- a/types/format_test.go +++ b/types/format_test.go @@ -119,7 +119,7 @@ func (s *testTimeSuite) TestStrToDate(c *C) { {`70/10/22`, `%Y/%m/%d`, types.FromDate(1970, 10, 22, 0, 0, 0, 0)}, {`18/10/22`, `%Y/%m/%d`, types.FromDate(2018, 10, 22, 0, 0, 0, 0)}, {`100/10/22`, `%Y/%m/%d`, types.FromDate(100, 10, 22, 0, 0, 0, 0)}, - // %X %V %W + // %X %V %W {`200442 Monday`, `%X%V %W`, types.FromDate(2004, 10, 18, 0, 0, 0, 0)}, {`2004420`, `%X%V%w`, types.FromDate(2004, 10, 17, 0, 0, 0, 0)}, {`2004423`, `%X%V%w`, types.FromDate(2004, 10, 20, 0, 0, 0, 0)}, @@ -185,7 +185,7 @@ func (s *testTimeSuite) TestStrToDate(c *C) { {"2010-11-12 11 am", `%Y-%m-%d %H %p`}, {"2010-11-12 13 am", `%Y-%m-%d %h %p`}, {"2010-11-12 0 am", `%Y-%m-%d %h %p`}, - // %X %V %W + // %X %V %W {`2004427`, `%X%V%w`}, // %w range 0-6, here is 7 {"2004421", "%x%V%w"}, // %x should be used with %v {"2004421", "%X%v%w"}, // %X should be used with %V