parser, infoschema, executor: Add information_schema.keywords (#48807)

.gitignore

@@ -37,3 +37,4 @@ bazel-testlogs
 bazel-tidb
 .ijwb/
 /oom_record/
+genkeyword

pkg/executor/builder.go

@@ -2117,7 +2117,8 @@ func (b *executorBuilder) buildMemTable(v *plannercore.PhysicalMemTable) exec.Ex
 			strings.ToLower(infoschema.ClusterTableMemoryUsageOpsHistory),
 			strings.ToLower(infoschema.TableResourceGroups),
 			strings.ToLower(infoschema.TableRunawayWatches),
-			strings.ToLower(infoschema.TableCheckConstraints):
+			strings.ToLower(infoschema.TableCheckConstraints),
+			strings.ToLower(infoschema.TableKeywords):
 			return &MemTableReaderExec{
 				BaseExecutor: exec.NewBaseExecutor(b.ctx, v.Schema(), v.ID()),
 				table:        v.Table,

pkg/executor/infoschema_reader.go

@@ -43,6 +43,7 @@ import (
 	"github.com/pingcap/tidb/pkg/infoschema"
 	"github.com/pingcap/tidb/pkg/kv"
 	"github.com/pingcap/tidb/pkg/meta/autoid"
+	"github.com/pingcap/tidb/pkg/parser"
 	"github.com/pingcap/tidb/pkg/parser/charset"
 	"github.com/pingcap/tidb/pkg/parser/model"
 	"github.com/pingcap/tidb/pkg/parser/mysql"
@@ -196,6 +197,8 @@ func (e *memtableRetriever) retrieve(ctx context.Context, sctx sessionctx.Contex
 			err = e.setDataFromRunawayWatches(sctx)
 		case infoschema.TableCheckConstraints:
 			err = e.setDataFromCheckConstraints(sctx, dbs)
+		case infoschema.TableKeywords:
+			err = e.setDataFromKeywords()
 		}
 		if err != nil {
 			return nil, err
@@ -3386,6 +3389,16 @@ func (e *memtableRetriever) setDataFromResourceGroups() error {
 	return nil
 }
 
+func (e *memtableRetriever) setDataFromKeywords() error {
+	rows := make([][]types.Datum, 0, len(parser.Keywords))
+	for _, kw := range parser.Keywords {
+		row := types.MakeDatums(kw.Word, kw.Reserved)
+		rows = append(rows, row)
+	}
+	e.rows = rows
+	return nil
+}
+
 func checkRule(rule *label.Rule) (dbName, tableName string, partitionName string, err error) {
 	s := strings.Split(rule.ID, "/")
 	if len(s) < 3 {

pkg/executor/infoschema_reader_internal_test.go

@@ -71,3 +71,11 @@ func TestSetDataFromCheckConstraints(t *testing.T) {
 	require.Equal(t, types.NewStringDatum("t2_c1"), mt.rows[0][2])
 	require.Equal(t, types.NewStringDatum("(id<10)"), mt.rows[0][3])
 }
+
+func TestSetDataFromKeywords(t *testing.T) {
+	mt := memtableRetriever{}
+	err := mt.setDataFromKeywords()
+	require.NoError(t, err)
+	require.Equal(t, types.NewStringDatum("ADD"), mt.rows[0][0]) // Keyword: ADD
+	require.Equal(t, types.NewIntDatum(1), mt.rows[0][1])        // Reserved: true(1)
+}

pkg/infoschema/tables.go

@@ -209,6 +209,8 @@ const (
 	TableRunawayWatches = "RUNAWAY_WATCHES"
 	// TableCheckConstraints is the list of CHECK constraints.
 	TableCheckConstraints = "CHECK_CONSTRAINTS"
+	// TableKeywords is the list of keywords.
+	TableKeywords = "KEYWORDS"
 )
 
 const (
@@ -318,6 +320,7 @@ var tableIDMap = map[string]int64{
 	TableResourceGroups:                  autoid.InformationSchemaDBID + 88,
 	TableRunawayWatches:                  autoid.InformationSchemaDBID + 89,
 	TableCheckConstraints:                autoid.InformationSchemaDBID + 90,
+	TableKeywords:                        autoid.InformationSchemaDBID + 92,
 }
 
 // columnInfo represents the basic column information of all kinds of INFORMATION_SCHEMA tables
@@ -1637,6 +1640,11 @@ var tableCheckConstraintsCols = []columnInfo{
 	{name: "CHECK_CLAUSE", tp: mysql.TypeLongBlob, size: types.UnspecifiedLength, flag: mysql.NotNullFlag},
 }
 
+var tableKeywords = []columnInfo{
+	{name: "WORD", tp: mysql.TypeVarchar, size: 128},
+	{name: "RESERVED", tp: mysql.TypeLong, size: 11},
+}
+
 // GetShardingInfo returns a nil or description string for the sharding information of given TableInfo.
 // The returned description string may be:
 //   - "NOT_SHARDED": for tables that SHARD_ROW_ID_BITS is not specified.
@@ -2155,6 +2163,7 @@ var tableNameToColumns = map[string][]columnInfo{
 	TableResourceGroups:                     tableResourceGroupsCols,
 	TableRunawayWatches:                     tableRunawayWatchListCols,
 	TableCheckConstraints:                   tableCheckConstraintsCols,
+	TableKeywords:                           tableKeywords,
 }
 
 func createInfoSchemaTable(_ autoid.Allocators, meta *model.TableInfo) (table.Table, error) {

pkg/parser/BUILD.bazel

@@ -9,8 +9,10 @@ go_library(
     name = "parser",
     srcs = [
         "digester.go",
+        "generate.go",
         "hintparser.go",
         "hintparserimpl.go",
+        "keywords.go",
         "lexer.go",
         "misc.go",
         "parser.go",
@@ -41,6 +43,7 @@ go_test(
         "consistent_test.go",
         "digester_test.go",
         "hintparser_test.go",
+        "keywords_test.go",
         "lexer_test.go",
         "main_test.go",
         "parser_test.go",

pkg/parser/Makefile

@@ -1,12 +1,18 @@
 .PHONY: all parser clean
 
-all: fmt parser
+all: fmt parser generate
 
 test: fmt parser
 	sh test.sh
 
 parser: parser.go hintparser.go
 
+genkeyword: generate_keyword/genkeyword.go
+	go build -C generate_keyword -o ../genkeyword
+
+generate: genkeyword parser.y
+	go generate
+
 %arser.go: prefix = $(@:parser.go=)
 %arser.go: %arser.y bin/goyacc
 	@echo "bin/goyacc -o $@ -p yy$(prefix) -t $(prefix)Parser $<"

pkg/parser/generate.go

@@ -0,0 +1,4 @@
+package parser
+
+// Generate keywords.go
+//go:generate ./genkeyword

pkg/parser/generate_keyword/BUILD.bazel

@@ -0,0 +1,23 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test")
+
+go_library(
+    name = "generate_keyword_lib",
+    srcs = ["genkeyword.go"],
+    importpath = "github.com/pingcap/tidb/pkg/parser/generate_keyword",
+    visibility = ["//visibility:private"],
+)
+
+go_binary(
+    name = "generate_keyword",
+    embed = [":generate_keyword_lib"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "generate_keyword_test",
+    timeout = "short",
+    srcs = ["genkeyword_test.go"],
+    embed = [":generate_keyword_lib"],
+    flaky = True,
+    deps = ["@com_github_stretchr_testify//require"],
+)

pkg/parser/generate_keyword/genkeyword.go

@@ -0,0 +1,146 @@
+// Copyright 2023 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"regexp"
+	"strings"
+)
+
+const (
+	reservedKeywordStart   = "The following tokens belong to ReservedKeyword"
+	unreservedkeywordStart = "The following tokens belong to UnReservedKeyword"
+	notKeywordStart        = "The following tokens belong to NotKeywordToken"
+	tiDBKeywordStart       = "The following tokens belong to TiDBKeyword"
+)
+
+const (
+	sectionNone = iota
+	sectionReservedKeyword
+	sectionUnreservedKeyword
+	sectionTiDBKeyword
+)
+
+const (
+	fileStart = `// Copyright 2023 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package parser
+
+// WARNING: This file is generated by 'genkeyword'
+
+// KeywordsType defines the attributes of keywords
+type KeywordsType struct {
+	Word     string
+	Reserved bool
+	Section  string
+}
+
+// Keywords is used for all keywords in TiDB
+var Keywords = []KeywordsType{
+`
+	fileEnd = `}
+`
+)
+
+var keywordRe *regexp.Regexp
+
+// parseLine extracts a keyword from a line of parser.y
+// returns an empty string if there is no match.
+//
+// example data:
+//
+//	add               "ADD"
+//
+// Note that all keywords except `TiDB_CURRENT_TSO` are fully uppercase.
+func parseLine(line string) string {
+	if keywordRe == nil {
+		keywordRe = regexp.MustCompile(`^\s+\w+\s+"(\w+)"$`)
+	}
+	m := keywordRe.FindStringSubmatch(line)
+	if len(m) != 2 {
+		return ""
+	}
+	return m[1]
+}
+
+func main() {
+	parserData, err := os.ReadFile("parser.y")
+	if err != nil {
+		log.Fatalf("Failed to open parser.y: %s", err)
+	}
+	keywordsFile, err := os.OpenFile("keywords.go", os.O_CREATE|os.O_WRONLY, 0600)
+	if err != nil {
+		log.Fatalf("Failed to create keywords.go: %s", err)
+	}
+	err = keywordsFile.Truncate(0)
+	if err != nil {
+		log.Fatalf("Failed to create keywords.go: %s", err)
+	}
+	_, err = keywordsFile.WriteString(fileStart)
+	if err != nil {
+		log.Fatalf("Failed to write fileStart to keywords.go: %s", err)
+	}
+
+	section := sectionNone
+	for _, line := range strings.Split(string(parserData), "\n") {
+		if line == "" { // Empty line indicates section end
+			section = sectionNone
+		} else if strings.Contains(line, reservedKeywordStart) {
+			section = sectionReservedKeyword
+		} else if strings.Contains(line, unreservedkeywordStart) {
+			section = sectionUnreservedKeyword
+		} else if strings.Contains(line, tiDBKeywordStart) {
+			section = sectionTiDBKeyword
+		} else if strings.Contains(line, notKeywordStart) {
+			section = sectionNone
+		}
+
+		switch section {
+		case sectionReservedKeyword:
+			word := parseLine(line)
+			if len(word) > 0 {
+				fmt.Fprintf(keywordsFile, "\t{\"%s\", true, \"reserved\"},\n", word)
+			}
+		case sectionTiDBKeyword:
+			word := parseLine(line)
+			if len(word) > 0 {
+				fmt.Fprintf(keywordsFile, "\t{\"%s\", false, \"tidb\"},\n", word)
+			}
+		case sectionUnreservedKeyword:
+			word := parseLine(line)
+			if len(word) > 0 {
+				fmt.Fprintf(keywordsFile, "\t{\"%s\", false, \"unreserved\"},\n", word)
+			}
+		}
+	}
+
+	_, err = keywordsFile.WriteString(fileEnd)
+	if err != nil {
+		log.Fatalf("Failed to write fileEnd to keywords.go: %s", err)
+	}
+}

pkg/parser/generate_keyword/genkeyword_test.go

@@ -0,0 +1,15 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseLine(t *testing.T) {
+	add := parseLine("	add               \"ADD\"")
+	require.Equal(t, add, "ADD")
+
+	tso := parseLine("	tidbCurrentTSO    \"TiDB_CURRENT_TSO\"")
+	require.Equal(t, tso, "TiDB_CURRENT_TSO")
+}