Rewrite special comment parser (#711) (#810)

Makefile

@@ -1,41 +1,26 @@
 .PHONY: all parser clean
 
-ARCH:="`uname -s`"
-MAC:="Darwin"
-LINUX:="Linux"
+all: fmt parser
 
-all: parser.go fmt
-
-test: parser.go fmt
+test: fmt parser
 	sh test.sh
 
-parser.go: parser.y
-	make parser
-
-parser: bin/goyacc
-	bin/goyacc -o /dev/null parser.y
-	bin/goyacc -o parser.go parser.y 2>&1 | egrep "(shift|reduce)/reduce" | awk '{print} END {if (NR > 0) {print "Find conflict in parser.y. Please check y.output for more information."; exit 1;}}'
-	rm -f y.output
-
-	@if [ $(ARCH) = $(LINUX) ]; \
-	then \
-		sed -i -e 's|//line.*||' -e 's/yyEofCode/yyEOFCode/' parser.go; \
-	elif [ $(ARCH) = $(MAC) ]; \
-	then \
-		/usr/bin/sed -i "" 's|//line.*||' parser.go; \
-		/usr/bin/sed -i "" 's/yyEofCode/yyEOFCode/' parser.go; \
-	fi
+parser: parser.go hintparser.go
 
-	@awk 'BEGIN{print "// Code generated by goyacc DO NOT EDIT."} {print $0}' parser.go > tmp_parser.go && mv tmp_parser.go parser.go;
+%arser.go: prefix = $(@:parser.go=)
+%arser.go: %arser.y bin/goyacc
+	@echo "bin/goyacc -o $@ -p yy$(prefix) -t $(prefix)Parser $<"
+	@bin/goyacc -o $@ -p yy$(prefix) -t $(prefix)Parser $< || ( rm -f $@ && echo 'Please check y.output for more information' && exit 1 )
+	@rm -f y.output
 
 bin/goyacc: goyacc/main.go
 	GO111MODULE=on go build -o bin/goyacc goyacc/main.go
 
-fmt:
+fmt: bin/goyacc
 	@echo "gofmt (simplify)"
-	@ gofmt -s -l -w . 2>&1 | awk '{print} END{if(NR>0) {exit 1}}'
+	@gofmt -s -l -w . 2>&1 | awk '{print} END{if(NR>0) {exit 1}}'
 
 clean:
 	go clean -i ./...
 	rm -rf *.out
-	rm parser.go
+	rm -f parser.go hintparser.go

ast/misc.go

@@ -2102,11 +2102,16 @@ type TableOptimizerHint struct {
 
 // HintTable is table in the hint. It may have query block info.
 type HintTable struct {
+	DBName    model.CIStr
 	TableName model.CIStr
 	QBName    model.CIStr
 }
 
 func (ht *HintTable) Restore(ctx *RestoreCtx) {
+	if ht.DBName.L != "" {
+		ctx.WriteName(ht.DBName.String())
+		ctx.WriteKeyWord(".")
+	}
 	ctx.WriteName(ht.TableName.String())
 	if ht.QBName.L != "" {
 		ctx.WriteKeyWord("@")

ast/misc_test.go

@@ -208,9 +208,11 @@ func (ts *testMiscSuite) TestUserSpec(c *C) {
 func (ts *testMiscSuite) TestTableOptimizerHintRestore(c *C) {
 	testCases := []NodeRestoreTestCase{
 		{"USE_INDEX(t1 c1)", "USE_INDEX(`t1` `c1`)"},
+		{"USE_INDEX(test.t1 c1)", "USE_INDEX(`test`.`t1` `c1`)"},
 		{"USE_INDEX(@sel_1 t1 c1)", "USE_INDEX(@`sel_1` `t1` `c1`)"},
 		{"USE_INDEX(t1@sel_1 c1)", "USE_INDEX(`t1`@`sel_1` `c1`)"},
 		{"IGNORE_INDEX(t1 c1)", "IGNORE_INDEX(`t1` `c1`)"},
+		{"USE_INDEX(test.t1@sel_1 c1)", "USE_INDEX(`test`.`t1`@`sel_1` `c1`)"},
 		{"IGNORE_INDEX(@sel_1 t1 c1)", "IGNORE_INDEX(@`sel_1` `t1` `c1`)"},
 		{"IGNORE_INDEX(t1@sel_1 c1)", "IGNORE_INDEX(`t1`@`sel_1` `c1`)"},
 		{"TIDB_SMJ(`t1`)", "TIDB_SMJ(`t1`)"},

digester.go

@@ -173,17 +173,7 @@ func (d *sqlDigester) normalize(sql string) {
 
 func (d *sqlDigester) reduceOptimizerHint(tok *token) (reduced bool) {
 	// ignore /*+..*/
-	if tok.tok == hintBegin {
-		for {
-			tok, _, _ := d.lexer.scan()
-			if tok == 0 || (tok == unicode.ReplacementChar && d.lexer.r.eof()) {
-				break
-			}
-			if tok == hintEnd {
-				reduced = true
-				break
-			}
-		}
+	if tok.tok == hintComment {
 		return
 	}
 

goyacc/main.go

@@ -162,6 +162,7 @@ var (
 	oResolved   = flag.Bool("ex", false, "explain how were conflicts resolved")
 	oXErrors    = flag.String("xe", "", "generate eXtra errors from examples source file")
 	oXErrorsGen = flag.String("xegen", "", "generate error from examples source file automatically from the grammar")
+	oParserType = flag.String("t", "Parser", "name of the parser in the generated yyParse() function")
 )
 
 func main() {
@@ -300,7 +301,7 @@ func main1(in string) (err error) {
 
 	p, err := y.ProcessFile(token.NewFileSet(), in, &y.Options{
 		//NoDefault:   *oNoDefault,
-		AllowConflicts: true,
+		AllowConflicts: false,
 		Closures:       *oClosures,
 		LA:             *oLA,
 		Reducible:      *oReducible,
@@ -364,7 +365,7 @@ func main1(in string) (err error) {
 
 	// ----------------------------------------------------------- Prologue
 	f := strutil.IndentFormatter(out, "\t")
-	mustFormat(f, "// CAUTION: Generated file - DO NOT EDIT.\n\n")
+	mustFormat(f, "// Code generated by goyacc DO NOT EDIT.\n\n")
 	mustFormat(f, "%s", injectImport(p.Prologue))
 	mustFormat(f, `
 type %[1]sSymType %i%s%u
@@ -546,13 +547,12 @@ func %[1]slex1(yylex %[1]sLexer, lval *%[1]sSymType) (n int) {
 	return n
 }
 
-func %[1]sParse(yylex %[1]sLexer, parser *Parser) int {
+func %[1]sParse(yylex %[1]sLexer, parser *%[5]s) int {
 	const yyError = %[2]d
 
 	yyEx, _ := yylex.(%[1]sLexerEx)
 	var yyn int
 	parser.yylval = %[1]sSymType{}
-	parser.yyVAL = %[1]sSymType{}
 	yyS := parser.cache
 
 	Nerrs := 0   /* number of errors */
@@ -580,13 +580,13 @@ ret1:
 yystack:
 	/* put a state and value onto the stack */
 	yyp++
-	if yyp >= len(yyS) {
+	if yyp+1 >= len(yyS) {
 		nyys := make([]%[1]sSymType, len(yyS)*2)
 		copy(nyys, yyS)
 		yyS = nyys
 		parser.cache = yyS
 	}
-	yyS[yyp] = parser.yyVAL
+	parser.yyVAL = &yyS[yyp+1]
 	yyS[yyp].yys = yystate
 
 yynewstate:
@@ -614,7 +614,7 @@ yynewstate:
 	switch {
 	case yyn > 0: // shift
 		yychar = -1
-		parser.yyVAL = parser.yylval
+		*parser.yyVAL = parser.yylval
 		yystate = yyn
 		yyshift = yyn
 		if %[1]sDebug >= 2 {
@@ -712,7 +712,7 @@ yynewstate:
 		yyS = nyys
 		parser.cache = yyS
 	}
-	parser.yyVAL = yyS[yyp+1]
+	parser.yyVAL = &yyS[yyp+1]
 
 	/* consult goto table to find next state */
 	exState := yystate
@@ -724,7 +724,7 @@ yynewstate:
 
 	switch r {%i
 `,
-		*oPref, errSym, *oDlvalf, *oDlval)
+		*oPref, errSym, *oDlvalf, *oDlval, *oParserType)
 	for r, rule := range p.Rules {
 		if rule.Action == nil {
 			continue
@@ -781,7 +781,7 @@ yynewstate:
 	mustFormat(f, `%u
 	}
 
-	if yyEx != nil && yyEx.Reduced(r, exState, &parser.yyVAL) {
+	if yyEx != nil && yyEx.Reduced(r, exState, parser.yyVAL) {
 		return -1
 	}
 	goto yystack /* stack new state and value */