improve syntax error message (pingcap#175)

circle.yml

@@ -1,7 +1,7 @@
 version: 2
 
 jobs:
-  build:
+  build-ut:
     docker:
       - image: golang:1.11
     working_directory: /go/src/github.com/pingcap/parser
@@ -19,12 +19,36 @@ jobs:
       - run:
           name: "Build & Test"
           command: make test
-      - run:
-          name: "Integration Test"
-          command: |
-            cd /go/src/github.com/pingcap/
-            git clone git@github.com:pingcap/tidb.git
-            cd tidb
-            rm go.sum
-            GO111MODULE=on go mod edit -replace github.com/pingcap/parser=github.com/${CIRCLE_PR_USERNAME:-$CIRCLE_PROJECT_USERNAME}/${CIRCLE_PR_REPONAME:-$CIRCLE_PROJECT_REPONAME}@$CIRCLE_SHA1
-            make test
+  build-integration:
+    docker:
+    - image: golang:1.11
+    working_directory: /go/src/github.com/pingcap/parser
+    steps:
+    - checkout
+    - run:
+        name: "Verify parser.go is up-to-date"
+        command: |
+          mv parser.go parser.go.committed
+          make parser
+          diff -u parser.go.committed parser.go
+    - run:
+        name: "Check code format"
+        command: make fmt
+    - run:
+        name: "Build"
+        command: make
+    - run:
+        name: "Integration Test"
+        command: |
+          cd /go/src/github.com/pingcap/
+          git clone git@github.com:pingcap/tidb.git
+          cd tidb
+          rm go.sum
+          GO111MODULE=on go mod edit -replace github.com/pingcap/parser=github.com/${CIRCLE_PR_USERNAME:-$CIRCLE_PROJECT_USERNAME}/${CIRCLE_PR_REPONAME:-$CIRCLE_PROJECT_REPONAME}@$CIRCLE_SHA1
+          make test
+workflows:
+  version: 2
+  build_and_test:
+    jobs:
+    - build-ut
+    - build-integration

lexer.go

@@ -16,6 +16,7 @@ package parser
 import (
 	"bytes"
 	"fmt"
+	"strconv"
 	"strings"
 	"unicode"
 	"unicode/utf8"
@@ -50,6 +51,10 @@ type Scanner struct {
 	// It may break the compatibility when support those keywords,
 	// because some application may already use them as identifiers.
 	supportWindowFunc bool
+
+	// lastScanOffset indicates last offset returned by scan().
+	// It's used to substring sql in syntax error message.
+	lastScanOffset int
 }
 
 type specialCommentScanner interface {
@@ -123,17 +128,14 @@ func (s *Scanner) stmtText() string {
 // Scanner satisfies yyLexer interface which need this function.
 func (s *Scanner) Errorf(format string, a ...interface{}) {
 	str := fmt.Sprintf(format, a...)
-	col := s.r.p.Col
-	startPos := s.stmtStartPos
-	if s.r.s[startPos] == '\n' {
-		startPos++
-		col--
-	}
-	val := s.r.s[startPos:]
+	val := s.r.s[s.lastScanOffset:]
+	var lenStr = ""
 	if len(val) > 2048 {
+		lenStr = "(total length " + strconv.Itoa(len(val)) + ")"
 		val = val[:2048]
 	}
-	err := fmt.Errorf("line %d column %d near \"%s\"%s (total length %d)", s.r.p.Line, col, val, str, len(s.r.s))
+	err := fmt.Errorf("line %d column %d near \"%s\"%s %s",
+		s.r.p.Line, s.r.p.Col, val, str, lenStr)
 	s.errs = append(s.errs, err)
 }
 
@@ -144,6 +146,7 @@ func (s *Scanner) Errorf(format string, a ...interface{}) {
 // return invalid tells parser that scanner meets illegal character.
 func (s *Scanner) Lex(v *yySymType) int {
 	tok, pos, lit := s.scan()
+	s.lastScanOffset = pos.Offset
 	v.offset = pos.Offset
 	v.ident = lit
 	if tok == identifier {

mysql/errname.go

@@ -79,7 +79,7 @@ var MySQLErrName = map[uint16]string{
 	ErrDupKeyName:                               "Duplicate key name '%-.192s'",
 	ErrDupEntry:                                 "Duplicate entry '%-.192s' for key %d",
 	ErrWrongFieldSpec:                           "Incorrect column specifier for column '%-.192s'",
-	ErrParse:                                    "%s near '%-.80s' at line %d",
+	ErrParse:                                    "%s %s",
 	ErrEmptyQuery:                               "Query was empty",
 	ErrNonuniqTable:                             "Not unique table/alias: '%-.192s'",
 	ErrInvalidDefault:                           "Invalid default value for '%-.192s'",

parser_test.go

@@ -1870,30 +1870,37 @@ func (s *testParserSuite) TestHintError(c *C) {
 	stmt, warns, err := parser.Parse("select /*+ tidb_unknow(T1,t2) */ c1, c2 from t1, t2 where t1.c1 = t2.c1", "", "")
 	c.Assert(err, IsNil)
 	c.Assert(len(warns), Equals, 1)
-	c.Assert(warns[0].Error(), Equals, "line 1 column 32 near \"select /*+ tidb_unknow(T1,t2) */ c1, c2 from t1, t2 where t1.c1 = t2.c1\" (total length 71)")
+	c.Assert(warns[0].Error(), Equals, "line 1 column 32 near \"tidb_unknow(T1,t2) */ c1, c2 from t1, t2 where t1.c1 = t2.c1\" ")
 	c.Assert(len(stmt[0].(*ast.SelectStmt).TableHints), Equals, 0)
 	stmt, warns, err = parser.Parse("select /*+ tidb_unknow(T1,t2, 1) TIDB_INLJ(t1, T2) */ c1, c2 from t1, t2 where t1.c1 = t2.c1", "", "")
 	c.Assert(len(stmt[0].(*ast.SelectStmt).TableHints), Equals, 0)
 	c.Assert(err, IsNil)
 	c.Assert(len(warns), Equals, 1)
-	c.Assert(warns[0].Error(), Equals, "line 1 column 53 near \"select /*+ tidb_unknow(T1,t2, 1) TIDB_INLJ(t1, T2) */ c1, c2 from t1, t2 where t1.c1 = t2.c1\" (total length 92)")
+	c.Assert(warns[0].Error(), Equals, "line 1 column 53 near \"tidb_unknow(T1,t2, 1) TIDB_INLJ(t1, T2) */ c1, c2 from t1, t2 where t1.c1 = t2.c1\" ")
 	stmt, _, err = parser.Parse("select c1, c2 from /*+ tidb_unknow(T1,t2) */ t1, t2 where t1.c1 = t2.c1", "", "")
 	c.Assert(err, NotNil)
 	stmt, _, err = parser.Parse("select1 /*+ TIDB_INLJ(t1, T2) */ c1, c2 from t1, t2 where t1.c1 = t2.c1", "", "")
 	c.Assert(err, NotNil)
+	c.Assert(err.Error(), Equals, "line 1 column 7 near \"select1 /*+ TIDB_INLJ(t1, T2) */ c1, c2 from t1, t2 where t1.c1 = t2.c1\" ")
 	stmt, _, err = parser.Parse("select /*+ TIDB_INLJ(t1, T2) */ c1, c2 fromt t1, t2 where t1.c1 = t2.c1", "", "")
 	c.Assert(err, NotNil)
+	c.Assert(err.Error(), Equals, "line 1 column 47 near \"t1, t2 where t1.c1 = t2.c1\" ")
 	_, _, err = parser.Parse("SELECT 1 FROM DUAL WHERE 1 IN (SELECT /*+ DEBUG_HINT3 */ 1)", "", "")
 	c.Assert(err, IsNil)
 }
 
 func (s *testParserSuite) TestErrorMsg(c *C) {
 	parser := New()
 	_, _, err := parser.Parse("select1 1", "", "")
-	c.Assert(err.Error(), Equals, "line 1 column 7 near \"select1 1\" (total length 9)")
-
-	_, _, err = parser.Parse("select a1 from t1\nwhere t1.a2 = 1;\nselect1 1", "", "")
-	c.Assert(err.Error(), Equals, "line 3 column 7 near \"select1 1\" (total length 44)")
+	c.Assert(err.Error(), Equals, "line 1 column 7 near \"select1 1\" ")
+	_, _, err = parser.Parse("select 1 from1 dual", "", "")
+	c.Assert(err.Error(), Equals, "line 1 column 19 near \"dual\" ")
+	_, _, err = parser.Parse("select * from t1 join t2 from t1.a = t2.a;", "", "")
+	c.Assert(err.Error(), Equals, "line 1 column 29 near \"from t1.a = t2.a;\" ")
+	_, _, err = parser.Parse("select * from t1 join t2 one t1.a = t2.a;", "", "")
+	c.Assert(err.Error(), Equals, "line 1 column 31 near \"t1.a = t2.a;\" ")
+	_, _, err = parser.Parse("select * from t1 join t2 on t1.a >>> t2.a;", "", "")
+	c.Assert(err.Error(), Equals, "line 1 column 36 near \"> t2.a;\" ")
 }
 
 func (s *testParserSuite) TestOptimizerHints(c *C) {
@@ -2079,9 +2086,9 @@ func (s *testParserSuite) TestComment(c *C) {
 
 func (s *testParserSuite) TestCommentErrMsg(c *C) {
 	table := []testErrMsgCase{
-		{"delete from t where a = 7 or 1=1/*' and b = 'p'", false, errors.New("[parser:1064]You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '/*' and b = 'p'' at line 1")},
-		{"delete from t where a = 7 or\n 1=1/*' and b = 'p'", false, errors.New("[parser:1064]You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '/*' and b = 'p'' at line 2")},
-		{"select 1/*", false, errors.New("[parser:1064]You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '/*' at line 1")},
+		{"delete from t where a = 7 or 1=1/*' and b = 'p'", false, errors.New("near '/*' and b = 'p'' at line 1")},
+		{"delete from t where a = 7 or\n 1=1/*' and b = 'p'", false, errors.New("near '/*' and b = 'p'' at line 2")},
+		{"select 1/*", false, errors.New("near '/*' at line 1")},
 		{"select 1/* comment */", false, nil},
 	}
 	s.RunErrMsgTest(c, table)

yy_parser.go

@@ -14,6 +14,7 @@
 package parser
 
 import (
+	"fmt"
 	"math"
 	"regexp"
 	"strconv"
@@ -159,7 +160,7 @@ func ParseErrorWith(errstr string, lineno int) error {
 	if len(errstr) > mysql.ErrTextLength {
 		errstr = errstr[:mysql.ErrTextLength]
 	}
-	return ErrParse.GenWithStackByArgs(mysql.MySQLErrName[mysql.ErrSyntax], errstr, lineno)
+	return fmt.Errorf("near '%-.80s' at line %d", errstr, lineno)
 }
 
 // The select statement is not at the end of the whole statement, if the last