Skip to content

Commit

Permalink
lexer: quote identifier when get sql digest (pingcap#1151)
Browse files Browse the repository at this point in the history 
* fix

* fix

* Update digester.go

Co-authored-by: kennytm <kennytm@gmail.com>

Co-authored-by: kennytm <kennytm@gmail.com>
  • Loading branch information
@xiongjiwei @kennytm
xiongjiwei and kennytm authored Feb 24, 2021
1 parent ae05c24 commit d2b1523
Show file tree
Hide file tree
Showing 2 changed files with 46 additions and 21 deletions.
26 changes: 25 additions & 1 deletion digester.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ import (
"sync"
"unicode"
"unsafe"

"github.com/pingcap/parser/charset"
)

// DigestHash generates the digest of statements.
Expand Down Expand Up @@ -159,14 +161,36 @@ func (d *sqlDigester) normalize(sql string) {

d.reduceLit(&currTok)

if currTok.tok == identifier {
if strings.HasPrefix(currTok.lit, "_") {
_, _, err := charset.GetCharsetInfo(currTok.lit[1:])
if err == nil {
currTok.tok = underscoreCS
goto APPEND
}
}

if tok1 := d.lexer.isTokenIdentifier(currTok.lit, pos.Offset); tok1 != 0 {
currTok.tok = tok1
}
}
APPEND:
d.tokens = append(d.tokens, currTok)
}
d.lexer.reset("")
for i, token := range d.tokens {
if token.tok == singleAtIdentifier {
d.buffer.WriteString("@")
d.buffer.WriteString(token.lit)
} else if token.tok == underscoreCS {
d.buffer.WriteString("(_charset)")
} else if token.tok == identifier {
d.buffer.WriteByte('`')
d.buffer.WriteString(token.lit)
d.buffer.WriteByte('`')
} else {
d.buffer.WriteString(token.lit)
}
d.buffer.WriteString(token.lit)
if i != len(d.tokens)-1 {
d.buffer.WriteRune(' ')
}
Expand Down
41 changes: 21 additions & 20 deletions digester_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,33 +28,34 @@ func (s *testSQLDigestSuite) TestNormalize(c *C) {
input string
expect string
}{
{"select _utf8mb4'123'", "select (_charset) ?"},
{"SELECT 1", "select ?"},
{"select * from b where id = 1", "select * from b where id = ?"},
{"select 1 from b where id in (1, 3, '3', 1, 2, 3, 4)", "select ? from b where id in ( ... )"},
{"select 1 from b where id in (1, a, 4)", "select ? from b where id in ( ? , a , ? )"},
{"select 1 from b order by 2", "select ? from b order by 2"},
{"select * from b where id = 1", "select * from `b` where `id` = ?"},
{"select 1 from b where id in (1, 3, '3', 1, 2, 3, 4)", "select ? from `b` where `id` in ( ... )"},
{"select 1 from b where id in (1, a, 4)", "select ? from `b` where `id` in ( ? , `a` , ? )"},
{"select 1 from b order by 2", "select ? from `b` order by 2"},
{"select /*+ a hint */ 1", "select ?"},
{"select /* a hint */ 1", "select ?"},
{"select truncate(1, 2)", "select truncate ( ... )"},
{"select -1 + - 2 + b - c + 0.2 + (-2) from c where d in (1, -2, +3)", "select ? + ? + b - c + ? + ( ? ) from c where d in ( ... )"},
{"select * from t where a <= -1 and b < -2 and c = -3 and c > -4 and c >= -5 and e is 1", "select * from t where a <= ? and b < ? and c = ? and c > ? and c >= ? and e is ?"},
{"select count(a), b from t group by 2", "select count ( a ) , b from t group by 2"},
{"select count(a), b, c from t group by 2, 3", "select count ( a ) , b , c from t group by 2 , 3"},
{"select count(a), b, c from t group by (2, 3)", "select count ( a ) , b , c from t group by ( 2 , 3 )"},
{"select a, b from t order by 1, 2", "select a , b from t order by 1 , 2"},
{"select count(*) from t", "select count ( ? ) from t"},
{"select * from t Force Index(kk)", "select * from t"},
{"select * from t USE Index(kk)", "select * from t"},
{"select * from t Ignore Index(kk)", "select * from t"},
{"select * from t1 straight_join t2 on t1.id=t2.id", "select * from t1 join t2 on t1 . id = t2 . id"},
{"select -1 + - 2 + b - c + 0.2 + (-2) from c where d in (1, -2, +3)", "select ? + ? + `b` - `c` + ? + ( ? ) from `c` where `d` in ( ... )"},
{"select * from t where a <= -1 and b < -2 and c = -3 and c > -4 and c >= -5 and e is 1", "select * from `t` where `a` <= ? and `b` < ? and `c` = ? and `c` > ? and `c` >= ? and `e` is ?"},
{"select count(a), b from t group by 2", "select count ( `a` ) , `b` from `t` group by 2"},
{"select count(a), b, c from t group by 2, 3", "select count ( `a` ) , `b` , `c` from `t` group by 2 , 3"},
{"select count(a), b, c from t group by (2, 3)", "select count ( `a` ) , `b` , `c` from `t` group by ( 2 , 3 )"},
{"select a, b from t order by 1, 2", "select `a` , `b` from `t` order by 1 , 2"},
{"select count(*) from t", "select count ( ? ) from `t`"},
{"select * from t Force Index(kk)", "select * from `t`"},
{"select * from t USE Index(kk)", "select * from `t`"},
{"select * from t Ignore Index(kk)", "select * from `t`"},
{"select * from t1 straight_join t2 on t1.id=t2.id", "select * from `t1` join `t2` on `t1` . `id` = `t2` . `id`"},
// test syntax error, it will be checked by parser, but it should not make normalize dead loop.
{"select * from t ignore index(", "select * from t ignore index"},
{"select * from t ignore index(", "select * from `t` ignore index"},
{"select /*+ ", "select "},
{"select * from 🥳", "select * from"},
{"select 1 / 2", "select ? / ?"},
{"select * from t where a = 40 limit ?, ?", "select * from t where a = ? limit ..."},
{"select * from t where a > ?", "select * from t where a > ?"},
{"select @a=b from t", "select @a = b from t"},
{"select * from t where a = 40 limit ?, ?", "select * from `t` where `a` = ? limit ..."},
{"select * from t where a > ?", "select * from `t` where `a` > ?"},
{"select @a=b from t", "select @a = `b` from `t`"},
}
for _, test := range tests {
normalized := parser.Normalize(test.input)
Expand All @@ -73,7 +74,7 @@ func (s *testSQLDigestSuite) TestNormalizeDigest(c *C) {
normalized string
digest string
}{
{"select 1 from b where id in (1, 3, '3', 1, 2, 3, 4)", "select ? from b where id in ( ... )", "f36161eef94dbfbd5e2f6b9a2f498a4c7facc6860621fbeb8084f63898275016"},
{"select 1 from b where id in (1, 3, '3', 1, 2, 3, 4)", "select ? from `b` where `id` in ( ... )", "e1c8cc2738f596dc24f15ef8eb55e0d902910d7298983496362a7b46dbc0b310"},
}
for _, test := range tests {
normalized, digest := parser.NormalizeDigest(test.sql)
Expand Down

0 comments on commit d2b1523

Please sign in to comment.