Skip to content

Commit

Permalink
Implement date, time and timestamp literals (#10921)
Browse files Browse the repository at this point in the history
* Implement date, time and timestamp literals

Signed-off-by: Dirkjan Bussink <d.bussink@gmail.com>

* feat: add a test that verifies we can use date, timestamp and time as column names

Signed-off-by: Manan Gupta <manan@planetscale.com>

* feat: add precedence rule for non-reserved-keywords that can also be used to type cast STRINGS

Signed-off-by: Manan Gupta <manan@planetscale.com>

* test: fix tpch test to not expect syntax error

Signed-off-by: Manan Gupta <manan@planetscale.com>

* feat: make parser

Signed-off-by: Manan Gupta <manan@planetscale.com>

* Add validate of date style literals

This adds additional validation of date, time and timestamp literals. It
uses the same validation the evalengine already uses today which is more
restricted than MySQL itself but seems good enough for now.

It consolidates the parsing into one place so if the syntax allowed is
extended, we only have to update one location.

Signed-off-by: Dirkjan Bussink <d.bussink@gmail.com>

* Implement more complete TIME parsing support

Signed-off-by: Dirkjan Bussink <d.bussink@gmail.com>

Co-authored-by: Manan Gupta <manan@planetscale.com>
  • Loading branch information
dbussink and GuptaManan100 authored Aug 8, 2022
1 parent 6d68fd5 commit 6450e49
Show file tree
Hide file tree
Showing 20 changed files with 7,356 additions and 6,818 deletions.
1 change: 1 addition & 0 deletions go/mysql/constants.go
Original file line number Diff line number Diff line change
Expand Up @@ -541,6 +541,7 @@ const (
ERInvalidCastToJSON = 3147
ERJSONValueTooBig = 3150
ERJSONDocumentTooDeep = 3157
ERWrongValue = 1525

ErrCantCreateGeometryObject = 1416
ErrGISDataWrongEndianess = 3055
Expand Down
1 change: 1 addition & 0 deletions go/mysql/sql_error.go
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,7 @@ var stateToMysqlCode = map[vterrors.State]struct {
vterrors.WrongNumberOfColumnsInSelect: {num: ERWrongNumberOfColumnsInSelect, state: SSWrongNumberOfColumns},
vterrors.WrongTypeForVar: {num: ERWrongTypeForVar, state: SSClientError},
vterrors.WrongValueForVar: {num: ERWrongValueForVar, state: SSClientError},
vterrors.WrongValue: {num: ERWrongValue, state: SSUnknownSQLState},
vterrors.WrongFieldWithGroup: {num: ERWrongFieldWithGroup, state: SSClientError},
vterrors.ServerNotAvailable: {num: ERServerIsntAvailable, state: SSNetError},
vterrors.CantDoThisInTransaction: {num: ERCantDoThisDuringAnTransaction, state: SSCantDoThisDuringAnTransaction},
Expand Down
18 changes: 18 additions & 0 deletions go/test/endtoend/vtgate/queries/misc/misc_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,3 +68,21 @@ func TestHexVals(t *testing.T) {
mcmp.AssertMatches(`select 1 + x'09', 2 + 0x9`, `[[UINT64(10) UINT64(11)]]`)
mcmp.AssertMatches(`select 1 + X'09', 2 + 0x9 from t1`, `[[UINT64(10) UINT64(11)]]`)
}

func TestDateTimeTimestampVals(t *testing.T) {
mcmp, closer := start(t)
defer closer()

mcmp.AssertMatches(`select date'2022-08-03'`, `[[DATE("2022-08-03")]]`)
mcmp.AssertMatches(`select time'12:34:56'`, `[[TIME("12:34:56")]]`)
mcmp.AssertMatches(`select timestamp'2012-12-31 11:30:45'`, `[[DATETIME("2012-12-31 11:30:45")]]`)
}

func TestInvalidDateTimeTimestampVals(t *testing.T) {
mcmp, closer := start(t)
defer closer()

mcmp.ExecAllowAndCompareError(`select date'2022'`)
mcmp.ExecAllowAndCompareError(`select time'12:34:56:78'`)
mcmp.ExecAllowAndCompareError(`select timestamp'2022'`)
}
6 changes: 6 additions & 0 deletions go/vt/sqlparser/ast_format.go
Original file line number Diff line number Diff line change
Expand Up @@ -1276,6 +1276,12 @@ func (node *Literal) Format(buf *TrackedBuffer) {
buf.astPrintf(node, "X'%s'", node.Val)
case BitVal:
buf.astPrintf(node, "B'%s'", node.Val)
case DateVal:
buf.astPrintf(node, "date'%s'", node.Val)
case TimeVal:
buf.astPrintf(node, "time'%s'", node.Val)
case TimestampVal:
buf.astPrintf(node, "timestamp'%s'", node.Val)
default:
panic("unexpected")
}
Expand Down
12 changes: 12 additions & 0 deletions go/vt/sqlparser/ast_format_fast.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

18 changes: 18 additions & 0 deletions go/vt/sqlparser/ast_funcs.go
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,9 @@ const (
HexNum
HexVal
BitVal
DateVal
TimeVal
TimestampVal
)

// queryOptimizerPrefix is the prefix of an optimizer hint comment.
Expand Down Expand Up @@ -516,6 +519,21 @@ func NewBitLiteral(in string) *Literal {
return &Literal{Type: BitVal, Val: in}
}

// NewDateLiteral builds a new Date.
func NewDateLiteral(in string) *Literal {
return &Literal{Type: DateVal, Val: in}
}

// NewTimeLiteral builds a new Date.
func NewTimeLiteral(in string) *Literal {
return &Literal{Type: TimeVal, Val: in}
}

// NewTimestampLiteral builds a new Date.
func NewTimestampLiteral(in string) *Literal {
return &Literal{Type: TimestampVal, Val: in}
}

// NewArgument builds a new ValArg.
func NewArgument(in string) Argument {
return Argument(in)
Expand Down
31 changes: 31 additions & 0 deletions go/vt/sqlparser/normalizer.go
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,24 @@ func (nz *normalizer) WalkSelect(cursor *Cursor) bool {
return nz.err == nil // only continue if we haven't found any errors
}

func validateLiteral(node *Literal) (err error) {
switch node.Type {
case DateVal:
_, err = ParseDate(node.Val)
case TimeVal:
_, err = ParseTime(node.Val)
case TimestampVal:
_, err = ParseDateTime(node.Val)
}
return err
}

func (nz *normalizer) convertLiteralDedup(node *Literal, cursor *Cursor) {
err := validateLiteral(node)
if err != nil {
nz.err = err
}

// If value is too long, don't dedup.
// Such values are most likely not for vindexes.
// We save a lot of CPU because we avoid building
Expand Down Expand Up @@ -146,6 +163,11 @@ func (nz *normalizer) convertLiteralDedup(node *Literal, cursor *Cursor) {

// convertLiteral converts an Literal without the dedup.
func (nz *normalizer) convertLiteral(node *Literal, cursor *Cursor) {
err := validateLiteral(node)
if err != nil {
nz.err = err
}

bval := SQLToBindvar(node)
if bval == nil {
return
Expand Down Expand Up @@ -223,6 +245,15 @@ func SQLToBindvar(node SQLNode) *querypb.BindVariable {
return nil
}
v, err = sqltypes.NewValue(sqltypes.HexNum, []byte(fmt.Sprintf("0x%x", ui)))
case DateVal:
v, err = sqltypes.NewValue(sqltypes.Date, node.Bytes())
case TimeVal:
v, err = sqltypes.NewValue(sqltypes.Time, node.Bytes())
case TimestampVal:
// This is actually a DATETIME MySQL type. The timestamp literal
// syntax is part of the SQL standard and MySQL DATETIME matches
// the type best.
v, err = sqltypes.NewValue(sqltypes.Datetime, node.Bytes())
default:
return nil
}
Expand Down
48 changes: 48 additions & 0 deletions go/vt/sqlparser/normalizer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ import (

"vitess.io/vitess/go/sqltypes"
querypb "vitess.io/vitess/go/vt/proto/query"
vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc"
"vitess.io/vitess/go/vt/vterrors"
)

func TestNormalize(t *testing.T) {
Expand Down Expand Up @@ -261,6 +263,27 @@ func TestNormalize(t *testing.T) {
"bv3": sqltypes.HexNumBindVariable([]byte("0xa")),
"bv4": sqltypes.HexNumBindVariable([]byte("0x7f")),
},
}, {
// DateVal should also be normalized
in: `select date'2022-08-06'`,
outstmt: `select :bv1 from dual`,
outbv: map[string]*querypb.BindVariable{
"bv1": sqltypes.ValueBindVariable(sqltypes.MakeTrusted(sqltypes.Date, []byte("2022-08-06"))),
},
}, {
// TimeVal should also be normalized
in: `select time'17:05:12'`,
outstmt: `select :bv1 from dual`,
outbv: map[string]*querypb.BindVariable{
"bv1": sqltypes.ValueBindVariable(sqltypes.MakeTrusted(sqltypes.Time, []byte("17:05:12"))),
},
}, {
// TimestampVal should also be normalized
in: `select timestamp'2022-08-06 17:05:12'`,
outstmt: `select :bv1 from dual`,
outbv: map[string]*querypb.BindVariable{
"bv1": sqltypes.ValueBindVariable(sqltypes.MakeTrusted(sqltypes.Datetime, []byte("2022-08-06 17:05:12"))),
},
}}
for _, tc := range testcases {
t.Run(tc.in, func(t *testing.T) {
Expand All @@ -275,6 +298,31 @@ func TestNormalize(t *testing.T) {
}
}

func TestNormalizeInvalidDates(t *testing.T) {
testcases := []struct {
in string
err error
}{{
in: "select date'foo'",
err: vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongValue, "incorrect DATE value: '%s'", "foo"),
}, {
in: "select time'foo'",
err: vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongValue, "incorrect TIME value: '%s'", "foo"),
}, {
in: "select timestamp'foo'",
err: vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongValue, "incorrect DATETIME value: '%s'", "foo"),
}}
for _, tc := range testcases {
t.Run(tc.in, func(t *testing.T) {
stmt, err := Parse(tc.in)
require.NoError(t, err)
known := GetBindvars(stmt)
bv := make(map[string]*querypb.BindVariable)
require.EqualError(t, Normalize(stmt, NewReservedVars("bv", known), bv), tc.err.Error())
})
}
}

func TestNormalizeValidSQL(t *testing.T) {
for _, tcase := range validSQL {
t.Run(tcase.input, func(t *testing.T) {
Expand Down
124 changes: 124 additions & 0 deletions go/vt/sqlparser/parse_date.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
/*
Copyright 2019 The Vitess Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package sqlparser

import (
"fmt"
"strconv"
"strings"
"time"

vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc"
"vitess.io/vitess/go/vt/vterrors"
)

var dateFormats = []string{"2006-01-02", "06-01-02", "20060102", "060102"}
var datetimeFormats = []string{"2006-01-02 15:04:05.9", "06-01-02 15:04:05.9", "20060102150405.9", "060102150405.9"}
var timeWithDayFormats = []string{"15:04:05.9", "15:04", "15"}
var timeWithoutDayFormats = []string{"15:04:05.9", "15:04", "150405.9", "0405", "05"}

func ParseDate(in string) (t time.Time, err error) {
for _, f := range dateFormats {
t, err = time.Parse(f, in)
if err == nil {
return t, nil
}
}
return t, vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongValue, "incorrect DATE value: '%s'", in)
}

func ParseTime(in string) (t time.Time, err error) {
// ParseTime is right now only excepting on specific
// time format and doesn't accept all formats MySQL accepts.
// Can be improved in the future as needed.
if in == "" {
return t, vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongValue, "incorrect TIME value: '%s'", in)
}
start := 0
neg := in[start] == '-'
if neg {
start++
}

parts := strings.Split(in[start:], " ")
if len(parts) > 2 {
return t, vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongValue, "incorrect TIME value: '%s'", in)
}
days := 0
hourMinuteSeconds := parts[0]
if len(parts) == 2 {
days, err = strconv.Atoi(parts[0])
if err != nil {
fmt.Printf("atoi failed: %+v\n", err)
return t, vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongValue, "incorrect TIME value: '%s'", in)
}
if days < 0 {
// Double negative which is not allowed
return t, vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongValue, "incorrect TIME value: '%s'", in)
}
if days > 34 {
return t, vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongValue, "incorrect TIME value: '%s'", in)
}
for _, f := range timeWithDayFormats {
t, err = time.Parse(f, parts[1])
if err == nil {
break
}
}
} else {
for _, f := range timeWithoutDayFormats {
t, err = time.Parse(f, hourMinuteSeconds)
if err == nil {
break
}
}
}

if err != nil {
return t, vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongValue, "incorrect TIME value: '%s'", in)
}

// setting the date to today's date, because t is "0000-01-01 xx:xx:xx"
now := time.Now()
year, month, day := now.Date()
if neg {
// If we have a negative time, we start with the start of today
// and substract the total duration of the parsed time.
today := time.Date(year, month, day, 0, 0, 0, 0, t.Location())
duration := time.Duration(days)*24*time.Hour +
time.Duration(t.Hour())*time.Hour +
time.Duration(t.Minute())*time.Minute +
time.Duration(t.Second())*time.Second +
time.Duration(t.Nanosecond())*time.Nanosecond
t = today.Add(-duration)
} else {
// In case of a positive time, we can take a quicker
// shortcut and add the date of today.
t = t.AddDate(year, int(month-1), day-1+days)
}
return t, nil
}

func ParseDateTime(in string) (t time.Time, err error) {
for _, f := range datetimeFormats {
t, err = time.Parse(f, in)
if err == nil {
return t, nil
}
}
return t, vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongValue, "incorrect DATETIME value: '%s'", in)
}
Loading

0 comments on commit 6450e49

Please sign in to comment.