From 6ccbcef9ebfc32fc733b949d4889fd4aa3e423b3 Mon Sep 17 00:00:00 2001 From: ti-srebot <66930949+ti-srebot@users.noreply.github.com> Date: Tue, 8 Mar 2022 16:07:49 +0800 Subject: [PATCH 1/7] executor: fix load data panic if the data is broken at escape character (#30868) (#31774) close pingcap/tidb#31589 --- executor/load_data.go | 135 +++++++++-------------------------------- executor/write_test.go | 24 +------- 2 files changed, 30 insertions(+), 129 deletions(-) diff --git a/executor/load_data.go b/executor/load_data.go index 1202675ebbce0..7d124c0cacf3d 100644 --- a/executor/load_data.go +++ b/executor/load_data.go @@ -363,65 +363,20 @@ func (e *LoadDataInfo) SetMaxRowsInBatch(limit uint64) { e.curBatchCnt = 0 } -// getValidData returns prevData and curData that starts from starting symbol. -// If the data doesn't have starting symbol, prevData is nil and curData is curData[len(curData)-startingLen+1:]. -// If curData size less than startingLen, curData is returned directly. -func (e *LoadDataInfo) getValidData(prevData, curData []byte) ([]byte, []byte) { - startingLen := len(e.LinesInfo.Starting) - if startingLen == 0 { - return prevData, curData - } - - prevLen := len(prevData) - if prevLen > 0 { - // starting symbol in the prevData - idx := strings.Index(string(hack.String(prevData)), e.LinesInfo.Starting) - if idx != -1 { - return prevData[idx:], curData - } - - // starting symbol in the middle of prevData and curData - restStart := curData - if len(curData) >= startingLen { - restStart = curData[:startingLen-1] - } - prevData = append(prevData, restStart...) - idx = strings.Index(string(hack.String(prevData)), e.LinesInfo.Starting) - if idx != -1 { - return prevData[idx:prevLen], curData - } - } - - // starting symbol in the curData +// getValidData returns curData that starts from starting symbol. +// If the data doesn't have starting symbol, return curData[len(curData)-startingLen+1:] and false. +func (e *LoadDataInfo) getValidData(curData []byte) ([]byte, bool) { idx := strings.Index(string(hack.String(curData)), e.LinesInfo.Starting) - if idx != -1 { - return nil, curData[idx:] + if idx == -1 { + return curData[len(curData)-len(e.LinesInfo.Starting)+1:], false } - // no starting symbol - if len(curData) >= startingLen { - curData = curData[len(curData)-startingLen+1:] - } - return nil, curData -} - -func (e *LoadDataInfo) isInQuoter(bs []byte) bool { - inQuoter := false - for i := 0; i < len(bs); i++ { - switch bs[i] { - case e.FieldsInfo.Enclosed: - inQuoter = !inQuoter - case e.FieldsInfo.Escaped: - i++ - default: - } - } - return inQuoter + return curData[idx:], true } -// IndexOfTerminator return index of terminator, if not, return -1. +// indexOfTerminator return index of terminator, if not, return -1. // normally, the field terminator and line terminator is short, so we just use brute force algorithm. -func (e *LoadDataInfo) IndexOfTerminator(bs []byte, inQuoter bool) int { +func (e *LoadDataInfo) indexOfTerminator(bs []byte) int { fieldTerm := []byte(e.FieldsInfo.Terminated) fieldTermLen := len(fieldTerm) lineTerm := []byte(e.LinesInfo.Terminated) @@ -459,15 +414,16 @@ func (e *LoadDataInfo) IndexOfTerminator(bs []byte, inQuoter bool) int { } } atFieldStart := true + inQuoter := false loop: for i := 0; i < len(bs); i++ { - if atFieldStart && bs[i] == e.FieldsInfo.Enclosed { + if atFieldStart && e.FieldsInfo.Enclosed != byte(0) && bs[i] == e.FieldsInfo.Enclosed { inQuoter = !inQuoter atFieldStart = false continue } restLen := len(bs) - i - 1 - if inQuoter && bs[i] == e.FieldsInfo.Enclosed { + if inQuoter && e.FieldsInfo.Enclosed != byte(0) && bs[i] == e.FieldsInfo.Enclosed { // look ahead to see if it is end of line or field. switch cmpTerm(restLen, bs[i+1:]) { case lineTermType: @@ -505,67 +461,32 @@ loop: // getLine returns a line, curData, the next data start index and a bool value. // If it has starting symbol the bool is true, otherwise is false. func (e *LoadDataInfo) getLine(prevData, curData []byte, ignore bool) ([]byte, []byte, bool) { - startingLen := len(e.LinesInfo.Starting) - prevData, curData = e.getValidData(prevData, curData) - if prevData == nil && len(curData) < startingLen { - return nil, curData, false - } - inquotor := e.isInQuoter(prevData) - prevLen := len(prevData) - terminatedLen := len(e.LinesInfo.Terminated) - curStartIdx := 0 - if prevLen < startingLen { - curStartIdx = startingLen - prevLen - } - endIdx := -1 - if len(curData) >= curStartIdx { - if ignore { - endIdx = strings.Index(string(hack.String(curData[curStartIdx:])), e.LinesInfo.Terminated) - } else { - endIdx = e.IndexOfTerminator(curData[curStartIdx:], inquotor) - } - } - if endIdx == -1 { - // no terminated symbol - if len(prevData) == 0 { - return nil, curData, true - } - - // terminated symbol in the middle of prevData and curData + if prevData != nil { curData = append(prevData, curData...) - if ignore { - endIdx = strings.Index(string(hack.String(curData[startingLen:])), e.LinesInfo.Terminated) - } else { - endIdx = e.IndexOfTerminator(curData[startingLen:], inquotor) + } + startLen := len(e.LinesInfo.Starting) + if startLen != 0 { + if len(curData) < startLen { + return nil, curData, false } - if endIdx != -1 { - nextDataIdx := startingLen + endIdx + terminatedLen - return curData[startingLen : startingLen+endIdx], curData[nextDataIdx:], true + var ok bool + curData, ok = e.getValidData(curData) + if !ok { + return nil, curData, false } - // no terminated symbol - return nil, curData, true - } - - // terminated symbol in the curData - nextDataIdx := curStartIdx + endIdx + terminatedLen - if len(prevData) == 0 { - return curData[curStartIdx : curStartIdx+endIdx], curData[nextDataIdx:], true } - - // terminated symbol in the curData - prevData = append(prevData, curData[:nextDataIdx]...) + var endIdx int if ignore { - endIdx = strings.Index(string(hack.String(prevData[startingLen:])), e.LinesInfo.Terminated) + endIdx = strings.Index(string(hack.String(curData[startLen:])), e.LinesInfo.Terminated) } else { - endIdx = e.IndexOfTerminator(prevData[startingLen:], inquotor) + endIdx = e.indexOfTerminator(curData[startLen:]) } - if endIdx >= prevLen { - return prevData[startingLen : startingLen+endIdx], curData[nextDataIdx:], true + + if endIdx == -1 { + return nil, curData, true } - // terminated symbol in the middle of prevData and curData - lineLen := startingLen + endIdx + terminatedLen - return prevData[startingLen : startingLen+endIdx], curData[lineLen-prevLen:], true + return curData[startLen : startLen+endIdx], curData[startLen+endIdx+len(e.LinesInfo.Terminated):], true } // InsertData inserts data into specified table according to the specified format. diff --git a/executor/write_test.go b/executor/write_test.go index ec326d0b6d436..2584408e5d7c8 100644 --- a/executor/write_test.go +++ b/executor/write_test.go @@ -24,7 +24,6 @@ import ( "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/executor" "github.com/pingcap/tidb/kv" - "github.com/pingcap/tidb/parser/ast" "github.com/pingcap/tidb/parser/model" "github.com/pingcap/tidb/parser/mysql" "github.com/pingcap/tidb/planner/core" @@ -2118,33 +2117,14 @@ func TestLoadDataEscape(t *testing.T) { {nil, []byte("7\trtn0ZbN\n"), []string{"7|" + string([]byte{'r', 't', 'n', '0', 'Z', 'b', 'N'})}, nil, trivialMsg}, {nil, []byte("8\trtn0Zb\\N\n"), []string{"8|" + string([]byte{'r', 't', 'n', '0', 'Z', 'b', 'N'})}, nil, trivialMsg}, {nil, []byte("9\ttab\\ tab\n"), []string{"9|tab tab"}, nil, trivialMsg}, + // data broken at escape character. + {[]byte("1\ta string\\"), []byte("\n1\n"), []string{"1|a string\n1"}, nil, trivialMsg}, } deleteSQL := "delete from load_data_test" selectSQL := "select * from load_data_test;" checkCases(tests, ld, t, tk, ctx, selectSQL, deleteSQL) } -func TestLoadDataWithLongContent(t *testing.T) { - e := &executor.LoadDataInfo{ - FieldsInfo: &ast.FieldsClause{Terminated: ",", Escaped: '\\', Enclosed: '"'}, - LinesInfo: &ast.LinesClause{Terminated: "\n"}, - } - tests := []struct { - content string - inQuoter bool - expectedIndex int - }{ - {"123,123\n123,123", false, 7}, - {"123123\\n123123", false, -1}, - {"123123\n123123", true, -1}, - {"123123\n123123\"\n", true, 14}, - } - - for _, tt := range tests { - require.Equal(t, tt.expectedIndex, e.IndexOfTerminator([]byte(tt.content), tt.inQuoter)) - } -} - // TestLoadDataSpecifiedColumns reuse TestLoadDataEscape's test case :-) func TestLoadDataSpecifiedColumns(t *testing.T) { trivialMsg := "Records: 1 Deleted: 0 Skipped: 0 Warnings: 0" From 8effbbcd180ecb703575cfb54f4ab9b1e74a12ed Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Wed, 23 Mar 2022 18:58:33 +0800 Subject: [PATCH 2/7] *: add pushdown for ShowStmt and implement for show columns (#31742) (#33342) close pingcap/tidb#29910 --- executor/builder.go | 1 + executor/show.go | 22 +- planner/cascades/implementation_rules.go | 6 +- planner/core/find_best_task.go | 2 +- planner/core/logical_plans.go | 2 + .../core/memtable_predicate_extractor_test.go | 363 ++++++++++-------- planner/core/physical_plans.go | 2 + planner/core/planbuilder.go | 8 + planner/core/show_predicate_extractor.go | 97 +++++ planner/core/stringer.go | 10 +- planner/core/stringer_test.go | 72 ++++ 11 files changed, 411 insertions(+), 174 deletions(-) create mode 100644 planner/core/show_predicate_extractor.go create mode 100644 planner/core/stringer_test.go diff --git a/executor/builder.go b/executor/builder.go index a718577cea424..19353296e5e8a 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -764,6 +764,7 @@ func (b *executorBuilder) buildShow(v *plannercore.PhysicalShow) Executor { IfNotExists: v.IfNotExists, GlobalScope: v.GlobalScope, Extended: v.Extended, + Extractor: v.Extractor, } if e.Tp == ast.ShowMasterStatus { // show master status need start ts. diff --git a/executor/show.go b/executor/show.go index 8449483a3787c..5116ea81056e8 100644 --- a/executor/show.go +++ b/executor/show.go @@ -20,6 +20,7 @@ import ( gjson "encoding/json" "fmt" "reflect" + "regexp" "sort" "strconv" "strings" @@ -84,6 +85,7 @@ type ShowExec struct { Flag int // Some flag parsed from sql, such as FULL. Roles []*auth.RoleIdentity // Used for show grants. User *auth.UserIdentity // Used by show grants, show create user. + Extractor plannercore.ShowPredicateExtractor is infoschema.InfoSchema @@ -514,10 +516,23 @@ func (e *ShowExec) fetchShowTableStatus(ctx context.Context) error { func (e *ShowExec) fetchShowColumns(ctx context.Context) error { tb, err := e.getTable() - if err != nil { return errors.Trace(err) } + var ( + fieldPatternsRegexp *regexp.Regexp + FieldFilterEnable bool + fieldFilter string + ) + if e.Extractor != nil { + extractor := (e.Extractor).(*plannercore.ShowColumnsTableExtractor) + if extractor.FieldPatterns != "" { + fieldPatternsRegexp = regexp.MustCompile(extractor.FieldPatterns) + } + FieldFilterEnable = extractor.Field != "" + fieldFilter = extractor.Field + } + checker := privilege.GetPrivilegeManager(e.ctx) activeRoles := e.ctx.GetSessionVars().ActiveRoles if checker != nil && e.ctx.GetSessionVars().User != nil && !checker.RequestVerification(activeRoles, e.DBName.O, tb.Meta().Name.O, "", mysql.InsertPriv|mysql.SelectPriv|mysql.UpdatePriv|mysql.ReferencesPriv) { @@ -536,10 +551,11 @@ func (e *ShowExec) fetchShowColumns(ctx context.Context) error { return err } for _, col := range cols { - if e.Column != nil && e.Column.Name.L != col.Name.L { + if FieldFilterEnable && col.Name.L != fieldFilter { + continue + } else if fieldPatternsRegexp != nil && !fieldPatternsRegexp.MatchString(col.Name.L) { continue } - desc := table.NewColDesc(col) var columnDefault interface{} if desc.DefaultValue != nil { diff --git a/planner/cascades/implementation_rules.go b/planner/cascades/implementation_rules.go index b686d17d72469..7b30b67287202 100644 --- a/planner/cascades/implementation_rules.go +++ b/planner/cascades/implementation_rules.go @@ -242,12 +242,14 @@ func (r *ImplShow) Match(expr *memo.GroupExpr, prop *property.PhysicalProperty) func (r *ImplShow) OnImplement(expr *memo.GroupExpr, reqProp *property.PhysicalProperty) ([]memo.Implementation, error) { logicProp := expr.Group.Prop show := expr.ExprNode.(*plannercore.LogicalShow) - // TODO(zz-jason): unifying LogicalShow and PhysicalShow to a single // struct. So that we don't need to create a new PhysicalShow object, which // can help us to reduce the gc pressure of golang runtime and improve the // overall performance. - showPhys := plannercore.PhysicalShow{ShowContents: show.ShowContents}.Init(show.SCtx()) + showPhys := plannercore.PhysicalShow{ + ShowContents: show.ShowContents, + Extractor: show.Extractor, + }.Init(show.SCtx()) showPhys.SetSchema(logicProp.Schema) return []memo.Implementation{impl.NewShowImpl(showPhys)}, nil } diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index ff90a92b9b497..c2535efd48db4 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -155,7 +155,7 @@ func (p *LogicalShow) findBestTask(prop *property.PhysicalProperty, planCounter if !prop.IsEmpty() || planCounter.Empty() { return invalidTask, 0, nil } - pShow := PhysicalShow{ShowContents: p.ShowContents}.Init(p.ctx) + pShow := PhysicalShow{ShowContents: p.ShowContents, Extractor: p.Extractor}.Init(p.ctx) pShow.SetSchema(p.schema) planCounter.Dec(1) return &rootTask{p: pShow}, 1, nil diff --git a/planner/core/logical_plans.go b/planner/core/logical_plans.go index c322f7e9489c1..79eb82a76b47a 100644 --- a/planner/core/logical_plans.go +++ b/planner/core/logical_plans.go @@ -1261,6 +1261,8 @@ type ShowContents struct { type LogicalShow struct { logicalSchemaProducer ShowContents + + Extractor ShowPredicateExtractor } // LogicalShowDDLJobs is for showing DDL job list. diff --git a/planner/core/memtable_predicate_extractor_test.go b/planner/core/memtable_predicate_extractor_test.go index 311b9ae4a8838..0fc8c4abf3743 100644 --- a/planner/core/memtable_predicate_extractor_test.go +++ b/planner/core/memtable_predicate_extractor_test.go @@ -18,57 +18,32 @@ import ( "context" "regexp" "sort" + "testing" "time" - . "github.com/pingcap/check" "github.com/pingcap/tidb/domain" - "github.com/pingcap/tidb/kv" + "github.com/pingcap/tidb/errno" "github.com/pingcap/tidb/parser" plannercore "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/session" - "github.com/pingcap/tidb/store/mockstore" + "github.com/pingcap/tidb/testkit" "github.com/pingcap/tidb/util/hint" "github.com/pingcap/tidb/util/set" + "github.com/pingcap/tidb/util/testutil" + "github.com/stretchr/testify/require" ) -var _ = Suite(&extractorSuite{}) - -type extractorSuite struct { - store kv.Storage - dom *domain.Domain -} - -func (s *extractorSuite) SetUpSuite(c *C) { - store, err := mockstore.NewMockStore() - c.Assert(err, IsNil) - c.Assert(store, NotNil) - - session.SetSchemaLease(0) - session.DisableStats4Test() - dom, err := session.BootstrapSession(store) - c.Assert(err, IsNil) - c.Assert(dom, NotNil) - - s.store = store - s.dom = dom -} - -func (s *extractorSuite) TearDownSuite(c *C) { - s.dom.Close() - s.store.Close() -} - -func (s *extractorSuite) getLogicalMemTable(c *C, se session.Session, parser *parser.Parser, sql string) *plannercore.LogicalMemTable { +func getLogicalMemTable(t *testing.T, dom *domain.Domain, se session.Session, parser *parser.Parser, sql string) *plannercore.LogicalMemTable { stmt, err := parser.ParseOneStmt(sql, "", "") - c.Assert(err, IsNil) + require.NoError(t, err) ctx := context.Background() - builder, _ := plannercore.NewPlanBuilder().Init(se, s.dom.InfoSchema(), &hint.BlockHintProcessor{}) + builder, _ := plannercore.NewPlanBuilder().Init(se, dom.InfoSchema(), &hint.BlockHintProcessor{}) plan, err := builder.Build(ctx, stmt) - c.Assert(err, IsNil) + require.NoError(t, err) logicalPlan, err := plannercore.LogicalOptimize(ctx, builder.GetOptFlag(), plan.(plannercore.LogicalPlan)) - c.Assert(err, IsNil) + require.NoError(t, err) // Obtain the leaf plan leafPlan := logicalPlan @@ -80,9 +55,12 @@ func (s *extractorSuite) getLogicalMemTable(c *C, se session.Session, parser *pa return logicalMemTable } -func (s *extractorSuite) TestClusterConfigTableExtractor(c *C) { - se, err := session.CreateSession4Test(s.store) - c.Assert(err, IsNil) +func TestClusterConfigTableExtractor(t *testing.T) { + store, dom, clean := testkit.CreateMockStoreAndDomain(t) + defer clean() + + se, err := session.CreateSession4Test(store) + require.NoError(t, err) parser := parser.New() var cases = []struct { @@ -236,25 +214,28 @@ func (s *extractorSuite) TestClusterConfigTableExtractor(c *C) { }, } for _, ca := range cases { - logicalMemTable := s.getLogicalMemTable(c, se, parser, ca.sql) - c.Assert(logicalMemTable.Extractor, NotNil) + logicalMemTable := getLogicalMemTable(t, dom, se, parser, ca.sql) + require.NotNil(t, logicalMemTable.Extractor) clusterConfigExtractor := logicalMemTable.Extractor.(*plannercore.ClusterTableExtractor) - c.Assert(clusterConfigExtractor.NodeTypes, DeepEquals, ca.nodeTypes, Commentf("SQL: %v", ca.sql)) - c.Assert(clusterConfigExtractor.Instances, DeepEquals, ca.instances, Commentf("SQL: %v", ca.sql)) - c.Assert(clusterConfigExtractor.SkipRequest, DeepEquals, ca.skipRequest, Commentf("SQL: %v", ca.sql)) + require.EqualValues(t, ca.nodeTypes, clusterConfigExtractor.NodeTypes, "SQL: %v", ca.sql) + require.EqualValues(t, ca.instances, clusterConfigExtractor.Instances, "SQL: %v", ca.sql) + require.EqualValues(t, ca.skipRequest, clusterConfigExtractor.SkipRequest, "SQL: %v", ca.sql) } } -func timestamp(c *C, s string) int64 { - t, err := time.ParseInLocation("2006-01-02 15:04:05.999", s, time.Local) - c.Assert(err, IsNil) - return t.UnixNano() / int64(time.Millisecond) +func timestamp(t *testing.T, s string) int64 { + tt, err := time.ParseInLocation("2006-01-02 15:04:05.999", s, time.Local) + require.NoError(t, err) + return tt.UnixNano() / int64(time.Millisecond) } -func (s *extractorSuite) TestClusterLogTableExtractor(c *C) { - se, err := session.CreateSession4Test(s.store) - c.Assert(err, IsNil) +func TestClusterLogTableExtractor(t *testing.T) { + store, dom, clean := testkit.CreateMockStoreAndDomain(t) + defer clean() + + se, err := session.CreateSession4Test(store) + require.NoError(t, err) parser := parser.New() var cases = []struct { @@ -414,77 +395,77 @@ func (s *extractorSuite) TestClusterLogTableExtractor(c *C) { sql: "select * from information_schema.cluster_log where time='2019-10-10 10:10:10'", nodeTypes: set.NewStringSet(), instances: set.NewStringSet(), - startTime: timestamp(c, "2019-10-10 10:10:10"), - endTime: timestamp(c, "2019-10-10 10:10:10"), + startTime: timestamp(t, "2019-10-10 10:10:10"), + endTime: timestamp(t, "2019-10-10 10:10:10"), }, { sql: "select * from information_schema.cluster_log where time>='2019-10-10 10:10:10' and time<='2019-10-11 10:10:10'", nodeTypes: set.NewStringSet(), instances: set.NewStringSet(), - startTime: timestamp(c, "2019-10-10 10:10:10"), - endTime: timestamp(c, "2019-10-11 10:10:10"), + startTime: timestamp(t, "2019-10-10 10:10:10"), + endTime: timestamp(t, "2019-10-11 10:10:10"), }, { sql: "select * from information_schema.cluster_log where time>'2019-10-10 10:10:10' and time<'2019-10-11 10:10:10'", nodeTypes: set.NewStringSet(), instances: set.NewStringSet(), - startTime: timestamp(c, "2019-10-10 10:10:10") + 1, - endTime: timestamp(c, "2019-10-11 10:10:10") - 1, + startTime: timestamp(t, "2019-10-10 10:10:10") + 1, + endTime: timestamp(t, "2019-10-11 10:10:10") - 1, }, { sql: "select * from information_schema.cluster_log where time>='2019-10-10 10:10:10' and time<'2019-10-11 10:10:10'", nodeTypes: set.NewStringSet(), instances: set.NewStringSet(), - startTime: timestamp(c, "2019-10-10 10:10:10"), - endTime: timestamp(c, "2019-10-11 10:10:10") - 1, + startTime: timestamp(t, "2019-10-10 10:10:10"), + endTime: timestamp(t, "2019-10-11 10:10:10") - 1, }, { sql: "select * from information_schema.cluster_log where time>='2019-10-12 10:10:10' and time<'2019-10-11 10:10:10'", nodeTypes: set.NewStringSet(), instances: set.NewStringSet(), - startTime: timestamp(c, "2019-10-12 10:10:10"), - endTime: timestamp(c, "2019-10-11 10:10:10") - 1, + startTime: timestamp(t, "2019-10-12 10:10:10"), + endTime: timestamp(t, "2019-10-11 10:10:10") - 1, skipRequest: true, }, { sql: "select * from information_schema.cluster_log where time>='2019-10-10 10:10:10'", nodeTypes: set.NewStringSet(), instances: set.NewStringSet(), - startTime: timestamp(c, "2019-10-10 10:10:10"), + startTime: timestamp(t, "2019-10-10 10:10:10"), }, { sql: "select * from information_schema.cluster_log where time>='2019-10-10 10:10:10' and time>='2019-10-11 10:10:10' and time>='2019-10-12 10:10:10'", nodeTypes: set.NewStringSet(), instances: set.NewStringSet(), - startTime: timestamp(c, "2019-10-12 10:10:10"), + startTime: timestamp(t, "2019-10-12 10:10:10"), }, { sql: "select * from information_schema.cluster_log where time>='2019-10-10 10:10:10' and time>='2019-10-11 10:10:10' and time>='2019-10-12 10:10:10' and time='2019-10-13 10:10:10'", nodeTypes: set.NewStringSet(), instances: set.NewStringSet(), - startTime: timestamp(c, "2019-10-13 10:10:10"), - endTime: timestamp(c, "2019-10-13 10:10:10"), + startTime: timestamp(t, "2019-10-13 10:10:10"), + endTime: timestamp(t, "2019-10-13 10:10:10"), }, { sql: "select * from information_schema.cluster_log where time<='2019-10-10 10:10:10' and time='2019-10-13 10:10:10'", nodeTypes: set.NewStringSet(), instances: set.NewStringSet(), - startTime: timestamp(c, "2019-10-13 10:10:10"), - endTime: timestamp(c, "2019-10-10 10:10:10"), + startTime: timestamp(t, "2019-10-13 10:10:10"), + endTime: timestamp(t, "2019-10-10 10:10:10"), skipRequest: true, }, { sql: "select * from information_schema.cluster_log where time='2019-10-10 10:10:10' and time<='2019-10-13 10:10:10'", nodeTypes: set.NewStringSet(), instances: set.NewStringSet(), - startTime: timestamp(c, "2019-10-10 10:10:10"), - endTime: timestamp(c, "2019-10-10 10:10:10"), + startTime: timestamp(t, "2019-10-10 10:10:10"), + endTime: timestamp(t, "2019-10-10 10:10:10"), }, { sql: "select * from information_schema.cluster_log where time>='2019-10-10 10:10:10' and message like '%a%'", nodeTypes: set.NewStringSet(), instances: set.NewStringSet(), - startTime: timestamp(c, "2019-10-10 10:10:10"), + startTime: timestamp(t, "2019-10-10 10:10:10"), patterns: []string{".*a.*"}, }, { @@ -531,34 +512,37 @@ func (s *extractorSuite) TestClusterLogTableExtractor(c *C) { }, } for _, ca := range cases { - logicalMemTable := s.getLogicalMemTable(c, se, parser, ca.sql) - c.Assert(logicalMemTable.Extractor, NotNil) + logicalMemTable := getLogicalMemTable(t, dom, se, parser, ca.sql) + require.NotNil(t, logicalMemTable.Extractor) clusterConfigExtractor := logicalMemTable.Extractor.(*plannercore.ClusterLogTableExtractor) - c.Assert(clusterConfigExtractor.NodeTypes, DeepEquals, ca.nodeTypes, Commentf("SQL: %v", ca.sql)) - c.Assert(clusterConfigExtractor.Instances, DeepEquals, ca.instances, Commentf("SQL: %v", ca.sql)) - c.Assert(clusterConfigExtractor.SkipRequest, DeepEquals, ca.skipRequest, Commentf("SQL: %v", ca.sql)) + require.EqualValues(t, ca.nodeTypes, clusterConfigExtractor.NodeTypes, "SQL: %v", ca.sql) + require.EqualValues(t, ca.instances, clusterConfigExtractor.Instances, "SQL: %v", ca.sql) + require.EqualValues(t, ca.skipRequest, clusterConfigExtractor.SkipRequest, "SQL: %v", ca.sql) if ca.startTime > 0 { - c.Assert(clusterConfigExtractor.StartTime, Equals, ca.startTime, Commentf("SQL: %v", ca.sql)) + require.Equal(t, ca.startTime, clusterConfigExtractor.StartTime, "SQL: %v", ca.sql) } if ca.endTime > 0 { - c.Assert(clusterConfigExtractor.EndTime, Equals, ca.endTime, Commentf("SQL: %v", ca.sql)) + require.Equal(t, ca.endTime, clusterConfigExtractor.EndTime, "SQL: %v", ca.sql) } - c.Assert(clusterConfigExtractor.Patterns, DeepEquals, ca.patterns, Commentf("SQL: %v", ca.sql)) + require.EqualValues(t, ca.patterns, clusterConfigExtractor.Patterns, "SQL: %v", ca.sql) if len(ca.level) > 0 { - c.Assert(clusterConfigExtractor.LogLevels, DeepEquals, ca.level, Commentf("SQL: %v", ca.sql)) + require.EqualValues(t, ca.level, clusterConfigExtractor.LogLevels, "SQL: %v", ca.sql) } } } -func (s *extractorSuite) TestMetricTableExtractor(c *C) { - se, err := session.CreateSession4Test(s.store) - c.Assert(err, IsNil) +func TestMetricTableExtractor(t *testing.T) { + store, dom, clean := testkit.CreateMockStoreAndDomain(t) + defer clean() + + se, err := session.CreateSession4Test(store) + require.NoError(t, err) - parseTime := func(c *C, s string) time.Time { - t, err := time.ParseInLocation(plannercore.MetricTableTimeFormat, s, time.Local) - c.Assert(err, IsNil) - return t + parseTime := func(t *testing.T, s string) time.Time { + tt, err := time.ParseInLocation(plannercore.MetricTableTimeFormat, s, time.Local) + require.NoError(t, err) + return tt } parser := parser.New() @@ -607,33 +591,33 @@ func (s *extractorSuite) TestMetricTableExtractor(c *C) { "instance": set.NewStringSet("127.0.0.1:10080"), "sql_type": set.NewStringSet("Update"), }, - startTime: parseTime(c, "2019-10-10 10:10:10"), - endTime: parseTime(c, "2019-10-10 10:10:10"), + startTime: parseTime(t, "2019-10-10 10:10:10"), + endTime: parseTime(t, "2019-10-10 10:10:10"), }, { sql: "select * from metrics_schema.tidb_query_duration where time>'2019-10-10 10:10:10' and time<'2019-10-11 10:10:10'", promQL: `histogram_quantile(0.9, sum(rate(tidb_server_handle_query_duration_seconds_bucket{}[60s])) by (le,sql_type,instance))`, - startTime: parseTime(c, "2019-10-10 10:10:10.001"), - endTime: parseTime(c, "2019-10-11 10:10:09.999"), + startTime: parseTime(t, "2019-10-10 10:10:10.001"), + endTime: parseTime(t, "2019-10-11 10:10:09.999"), }, { sql: "select * from metrics_schema.tidb_query_duration where time>='2019-10-10 10:10:10'", promQL: `histogram_quantile(0.9, sum(rate(tidb_server_handle_query_duration_seconds_bucket{}[60s])) by (le,sql_type,instance))`, - startTime: parseTime(c, "2019-10-10 10:10:10"), - endTime: parseTime(c, "2019-10-10 10:20:10"), + startTime: parseTime(t, "2019-10-10 10:10:10"), + endTime: parseTime(t, "2019-10-10 10:20:10"), }, { sql: "select * from metrics_schema.tidb_query_duration where time>='2019-10-10 10:10:10' and time<='2019-10-09 10:10:10'", promQL: "", - startTime: parseTime(c, "2019-10-10 10:10:10"), - endTime: parseTime(c, "2019-10-09 10:10:10"), + startTime: parseTime(t, "2019-10-10 10:10:10"), + endTime: parseTime(t, "2019-10-09 10:10:10"), skipRequest: true, }, { sql: "select * from metrics_schema.tidb_query_duration where time<='2019-10-09 10:10:10'", promQL: "histogram_quantile(0.9, sum(rate(tidb_server_handle_query_duration_seconds_bucket{}[60s])) by (le,sql_type,instance))", - startTime: parseTime(c, "2019-10-09 10:00:10"), - endTime: parseTime(c, "2019-10-09 10:10:10"), + startTime: parseTime(t, "2019-10-09 10:00:10"), + endTime: parseTime(t, "2019-10-09 10:10:10"), }, { sql: "select * from metrics_schema.tidb_query_duration where quantile=0.9 or quantile=0.8", @@ -649,34 +633,37 @@ func (s *extractorSuite) TestMetricTableExtractor(c *C) { } se.GetSessionVars().StmtCtx.TimeZone = time.Local for _, ca := range cases { - logicalMemTable := s.getLogicalMemTable(c, se, parser, ca.sql) - c.Assert(logicalMemTable.Extractor, NotNil) + logicalMemTable := getLogicalMemTable(t, dom, se, parser, ca.sql) + require.NotNil(t, logicalMemTable.Extractor) metricTableExtractor := logicalMemTable.Extractor.(*plannercore.MetricTableExtractor) if len(ca.labelConditions) > 0 { - c.Assert(metricTableExtractor.LabelConditions, DeepEquals, ca.labelConditions, Commentf("SQL: %v", ca.sql)) + require.EqualValues(t, ca.labelConditions, metricTableExtractor.LabelConditions, "SQL: %v", ca.sql) } - c.Assert(metricTableExtractor.SkipRequest, DeepEquals, ca.skipRequest, Commentf("SQL: %v", ca.sql)) + require.EqualValues(t, ca.skipRequest, metricTableExtractor.SkipRequest, "SQL: %v", ca.sql) if len(metricTableExtractor.Quantiles) > 0 { - c.Assert(metricTableExtractor.Quantiles, DeepEquals, ca.quantiles) + require.EqualValues(t, ca.quantiles, metricTableExtractor.Quantiles) } if !ca.skipRequest { promQL := metricTableExtractor.GetMetricTablePromQL(se, "tidb_query_duration") - c.Assert(promQL, DeepEquals, ca.promQL, Commentf("SQL: %v", ca.sql)) + require.EqualValues(t, promQL, ca.promQL, "SQL: %v", ca.sql) start, end := metricTableExtractor.StartTime, metricTableExtractor.EndTime - c.Assert(start.UnixNano() <= end.UnixNano(), IsTrue) + require.GreaterOrEqual(t, end.UnixNano(), start.UnixNano()) if ca.startTime.Unix() > 0 { - c.Assert(metricTableExtractor.StartTime, DeepEquals, ca.startTime, Commentf("SQL: %v, start_time: %v", ca.sql, metricTableExtractor.StartTime)) + require.EqualValues(t, ca.startTime, metricTableExtractor.StartTime, "SQL: %v, start_time: %v", ca.sql, metricTableExtractor.StartTime) } if ca.endTime.Unix() > 0 { - c.Assert(metricTableExtractor.EndTime, DeepEquals, ca.endTime, Commentf("SQL: %v, end_time: %v", ca.sql, metricTableExtractor.EndTime)) + require.EqualValues(t, ca.endTime, metricTableExtractor.EndTime, "SQL: %v, end_time: %v", ca.sql, metricTableExtractor.EndTime) } } } } -func (s *extractorSuite) TestMetricsSummaryTableExtractor(c *C) { - se, err := session.CreateSession4Test(s.store) - c.Assert(err, IsNil) +func TestMetricsSummaryTableExtractor(t *testing.T) { + store, dom, clean := testkit.CreateMockStoreAndDomain(t) + defer clean() + + se, err := session.CreateSession4Test(store) + require.NoError(t, err) var cases = []struct { sql string @@ -758,23 +745,26 @@ func (s *extractorSuite) TestMetricsSummaryTableExtractor(c *C) { for _, ca := range cases { sort.Float64s(ca.quantiles) - logicalMemTable := s.getLogicalMemTable(c, se, parser, ca.sql) - c.Assert(logicalMemTable.Extractor, NotNil) + logicalMemTable := getLogicalMemTable(t, dom, se, parser, ca.sql) + require.NotNil(t, logicalMemTable.Extractor) extractor := logicalMemTable.Extractor.(*plannercore.MetricSummaryTableExtractor) if len(ca.quantiles) > 0 { - c.Assert(extractor.Quantiles, DeepEquals, ca.quantiles, Commentf("SQL: %v", ca.sql)) + require.EqualValues(t, ca.quantiles, extractor.Quantiles, "SQL: %v", ca.sql) } if len(ca.names) > 0 { - c.Assert(extractor.MetricsNames, DeepEquals, ca.names, Commentf("SQL: %v", ca.sql)) + require.EqualValues(t, ca.names, extractor.MetricsNames, "SQL: %v", ca.sql) } - c.Assert(extractor.SkipRequest, Equals, ca.skipRequest, Commentf("SQL: %v", ca.sql)) + require.Equal(t, ca.skipRequest, extractor.SkipRequest, "SQL: %v", ca.sql) } } -func (s *extractorSuite) TestInspectionResultTableExtractor(c *C) { - se, err := session.CreateSession4Test(s.store) - c.Assert(err, IsNil) +func TestInspectionResultTableExtractor(t *testing.T) { + store, dom, clean := testkit.CreateMockStoreAndDomain(t) + defer clean() + + se, err := session.CreateSession4Test(store) + require.NoError(t, err) var cases = []struct { sql string @@ -897,23 +887,26 @@ func (s *extractorSuite) TestInspectionResultTableExtractor(c *C) { } parser := parser.New() for _, ca := range cases { - logicalMemTable := s.getLogicalMemTable(c, se, parser, ca.sql) - c.Assert(logicalMemTable.Extractor, NotNil) + logicalMemTable := getLogicalMemTable(t, dom, se, parser, ca.sql) + require.NotNil(t, logicalMemTable.Extractor) clusterConfigExtractor := logicalMemTable.Extractor.(*plannercore.InspectionResultTableExtractor) if len(ca.rules) > 0 { - c.Assert(clusterConfigExtractor.Rules, DeepEquals, ca.rules, Commentf("SQL: %v", ca.sql)) + require.EqualValues(t, ca.rules, clusterConfigExtractor.Rules, "SQL: %v", ca.sql) } if len(ca.items) > 0 { - c.Assert(clusterConfigExtractor.Items, DeepEquals, ca.items, Commentf("SQL: %v", ca.sql)) + require.EqualValues(t, ca.items, clusterConfigExtractor.Items, "SQL: %v", ca.sql) } - c.Assert(clusterConfigExtractor.SkipInspection, Equals, ca.skipInspection, Commentf("SQL: %v", ca.sql)) + require.Equal(t, ca.skipInspection, clusterConfigExtractor.SkipInspection, "SQL: %v", ca.sql) } } -func (s *extractorSuite) TestInspectionSummaryTableExtractor(c *C) { - se, err := session.CreateSession4Test(s.store) - c.Assert(err, IsNil) +func TestInspectionSummaryTableExtractor(t *testing.T) { + store, dom, clean := testkit.CreateMockStoreAndDomain(t) + defer clean() + + se, err := session.CreateSession4Test(store) + require.NoError(t, err) var cases = []struct { sql string @@ -996,23 +989,26 @@ func (s *extractorSuite) TestInspectionSummaryTableExtractor(c *C) { } parser := parser.New() for _, ca := range cases { - logicalMemTable := s.getLogicalMemTable(c, se, parser, ca.sql) - c.Assert(logicalMemTable.Extractor, NotNil) + logicalMemTable := getLogicalMemTable(t, dom, se, parser, ca.sql) + require.NotNil(t, logicalMemTable.Extractor) clusterConfigExtractor := logicalMemTable.Extractor.(*plannercore.InspectionSummaryTableExtractor) if len(ca.rules) > 0 { - c.Assert(clusterConfigExtractor.Rules, DeepEquals, ca.rules, Commentf("SQL: %v", ca.sql)) + require.EqualValues(t, ca.rules, clusterConfigExtractor.Rules, "SQL: %v", ca.sql) } if len(ca.names) > 0 { - c.Assert(clusterConfigExtractor.MetricNames, DeepEquals, ca.names, Commentf("SQL: %v", ca.sql)) + require.EqualValues(t, ca.names, clusterConfigExtractor.MetricNames, "SQL: %v", ca.sql) } - c.Assert(clusterConfigExtractor.SkipInspection, Equals, ca.skipInspection, Commentf("SQL: %v", ca.sql)) + require.Equal(t, ca.skipInspection, clusterConfigExtractor.SkipInspection, "SQL: %v", ca.sql) } } -func (s *extractorSuite) TestInspectionRuleTableExtractor(c *C) { - se, err := session.CreateSession4Test(s.store) - c.Assert(err, IsNil) +func TestInspectionRuleTableExtractor(t *testing.T) { + store, dom, clean := testkit.CreateMockStoreAndDomain(t) + defer clean() + + se, err := session.CreateSession4Test(store) + require.NoError(t, err) var cases = []struct { sql string @@ -1037,20 +1033,23 @@ func (s *extractorSuite) TestInspectionRuleTableExtractor(c *C) { } parser := parser.New() for _, ca := range cases { - logicalMemTable := s.getLogicalMemTable(c, se, parser, ca.sql) - c.Assert(logicalMemTable.Extractor, NotNil) + logicalMemTable := getLogicalMemTable(t, dom, se, parser, ca.sql) + require.NotNil(t, logicalMemTable.Extractor) clusterConfigExtractor := logicalMemTable.Extractor.(*plannercore.InspectionRuleTableExtractor) if len(ca.tps) > 0 { - c.Assert(clusterConfigExtractor.Types, DeepEquals, ca.tps, Commentf("SQL: %v", ca.sql)) + require.EqualValues(t, ca.tps, clusterConfigExtractor.Types, "SQL: %v", ca.sql) } - c.Assert(clusterConfigExtractor.SkipRequest, Equals, ca.skip, Commentf("SQL: %v", ca.sql)) + require.Equal(t, ca.skip, clusterConfigExtractor.SkipRequest, "SQL: %v", ca.sql) } } -func (s *extractorSuite) TestTiDBHotRegionsHistoryTableExtractor(c *C) { - se, err := session.CreateSession4Test(s.store) - c.Assert(err, IsNil) +func TestTiDBHotRegionsHistoryTableExtractor(t *testing.T) { + store, dom, clean := testkit.CreateMockStoreAndDomain(t) + defer clean() + + se, err := session.CreateSession4Test(store) + require.NoError(t, err) se.GetSessionVars().StmtCtx.TimeZone = time.Local var cases = []struct { @@ -1072,40 +1071,40 @@ func (s *extractorSuite) TestTiDBHotRegionsHistoryTableExtractor(c *C) { }, { sql: "select * from information_schema.tidb_hot_regions_history where update_time='2019-10-10 10:10:10'", - startTime: timestamp(c, "2019-10-10 10:10:10"), - endTime: timestamp(c, "2019-10-10 10:10:10"), + startTime: timestamp(t, "2019-10-10 10:10:10"), + endTime: timestamp(t, "2019-10-10 10:10:10"), isLearners: []bool{false, true}, isLeaders: []bool{false, true}, hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), }, { sql: "select * from information_schema.tidb_hot_regions_history where update_time>='2019-10-10 10:10:10' and update_time<='2019-10-11 10:10:10'", - startTime: timestamp(c, "2019-10-10 10:10:10"), - endTime: timestamp(c, "2019-10-11 10:10:10"), + startTime: timestamp(t, "2019-10-10 10:10:10"), + endTime: timestamp(t, "2019-10-11 10:10:10"), isLearners: []bool{false, true}, isLeaders: []bool{false, true}, hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), }, { sql: "select * from information_schema.tidb_hot_regions_history where update_time>'2019-10-10 10:10:10' and update_time<'2019-10-11 10:10:10'", - startTime: timestamp(c, "2019-10-10 10:10:10") + 1, - endTime: timestamp(c, "2019-10-11 10:10:10") - 1, + startTime: timestamp(t, "2019-10-10 10:10:10") + 1, + endTime: timestamp(t, "2019-10-11 10:10:10") - 1, isLearners: []bool{false, true}, isLeaders: []bool{false, true}, hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), }, { sql: "select * from information_schema.tidb_hot_regions_history where update_time>='2019-10-10 10:10:10' and update_time<'2019-10-11 10:10:10'", - startTime: timestamp(c, "2019-10-10 10:10:10"), - endTime: timestamp(c, "2019-10-11 10:10:10") - 1, + startTime: timestamp(t, "2019-10-10 10:10:10"), + endTime: timestamp(t, "2019-10-11 10:10:10") - 1, isLearners: []bool{false, true}, isLeaders: []bool{false, true}, hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), }, { sql: "select * from information_schema.tidb_hot_regions_history where update_time>='2019-10-12 10:10:10' and update_time<'2019-10-11 10:10:10'", - startTime: timestamp(c, "2019-10-12 10:10:10"), - endTime: timestamp(c, "2019-10-11 10:10:10") - 1, + startTime: timestamp(t, "2019-10-12 10:10:10"), + endTime: timestamp(t, "2019-10-11 10:10:10") - 1, isLearners: []bool{false, true}, isLeaders: []bool{false, true}, hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), @@ -1113,30 +1112,30 @@ func (s *extractorSuite) TestTiDBHotRegionsHistoryTableExtractor(c *C) { }, { sql: "select * from information_schema.tidb_hot_regions_history where update_time>='2019-10-10 10:10:10'", - startTime: timestamp(c, "2019-10-10 10:10:10"), + startTime: timestamp(t, "2019-10-10 10:10:10"), isLearners: []bool{false, true}, isLeaders: []bool{false, true}, hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), }, { sql: "select * from information_schema.tidb_hot_regions_history where update_time>='2019-10-10 10:10:10' and update_time>='2019-10-11 10:10:10' and update_time>='2019-10-12 10:10:10'", - startTime: timestamp(c, "2019-10-12 10:10:10"), + startTime: timestamp(t, "2019-10-12 10:10:10"), isLearners: []bool{false, true}, isLeaders: []bool{false, true}, hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), }, { sql: "select * from information_schema.tidb_hot_regions_history where update_time>='2019-10-10 10:10:10' and update_time>='2019-10-11 10:10:10' and update_time>='2019-10-12 10:10:10' and update_time='2019-10-13 10:10:10'", - startTime: timestamp(c, "2019-10-13 10:10:10"), - endTime: timestamp(c, "2019-10-13 10:10:10"), + startTime: timestamp(t, "2019-10-13 10:10:10"), + endTime: timestamp(t, "2019-10-13 10:10:10"), isLearners: []bool{false, true}, isLeaders: []bool{false, true}, hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), }, { sql: "select * from information_schema.tidb_hot_regions_history where update_time<='2019-10-10 10:10:10' and update_time='2019-10-13 10:10:10'", - startTime: timestamp(c, "2019-10-13 10:10:10"), - endTime: timestamp(c, "2019-10-10 10:10:10"), + startTime: timestamp(t, "2019-10-13 10:10:10"), + endTime: timestamp(t, "2019-10-10 10:10:10"), isLearners: []bool{false, true}, isLeaders: []bool{false, true}, hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), @@ -1144,8 +1143,8 @@ func (s *extractorSuite) TestTiDBHotRegionsHistoryTableExtractor(c *C) { }, { sql: "select * from information_schema.tidb_hot_regions_history where update_time='2019-10-10 10:10:10' and update_time<='2019-10-13 10:10:10'", - startTime: timestamp(c, "2019-10-10 10:10:10"), - endTime: timestamp(c, "2019-10-10 10:10:10"), + startTime: timestamp(t, "2019-10-10 10:10:10"), + endTime: timestamp(t, "2019-10-10 10:10:10"), isLearners: []bool{false, true}, isLeaders: []bool{false, true}, hotRegionTypes: set.NewStringSet(plannercore.HotRegionTypeRead, plannercore.HotRegionTypeWrite), @@ -1389,34 +1388,64 @@ func (s *extractorSuite) TestTiDBHotRegionsHistoryTableExtractor(c *C) { parser := parser.New() for _, ca := range cases { - logicalMemTable := s.getLogicalMemTable(c, se, parser, ca.sql) - c.Assert(logicalMemTable.Extractor, NotNil, Commentf("SQL: %v", ca.sql)) + logicalMemTable := getLogicalMemTable(t, dom, se, parser, ca.sql) + require.NotNil(t, logicalMemTable.Extractor, "SQL: %v", ca.sql) hotRegionsHistoryExtractor := logicalMemTable.Extractor.(*plannercore.HotRegionsHistoryTableExtractor) if ca.startTime > 0 { - c.Assert(hotRegionsHistoryExtractor.StartTime, Equals, ca.startTime, Commentf("SQL: %v", ca.sql)) + require.Equal(t, ca.startTime, hotRegionsHistoryExtractor.StartTime, "SQL: %v", ca.sql) } if ca.endTime > 0 { - c.Assert(hotRegionsHistoryExtractor.EndTime, Equals, ca.endTime, Commentf("SQL: %v", ca.sql)) + require.Equal(t, ca.endTime, hotRegionsHistoryExtractor.EndTime, "SQL: %v", ca.sql) } - c.Assert(hotRegionsHistoryExtractor.SkipRequest, DeepEquals, ca.skipRequest, Commentf("SQL: %v", ca.sql)) + require.EqualValues(t, ca.skipRequest, hotRegionsHistoryExtractor.SkipRequest, "SQL: %v", ca.sql) if len(ca.isLearners) > 0 { - c.Assert(hotRegionsHistoryExtractor.IsLearners, DeepEquals, ca.isLearners, Commentf("SQL: %v", ca.sql)) + require.EqualValues(t, ca.isLearners, hotRegionsHistoryExtractor.IsLearners, "SQL: %v", ca.sql) } if len(ca.isLeaders) > 0 { - c.Assert(hotRegionsHistoryExtractor.IsLeaders, DeepEquals, ca.isLeaders, Commentf("SQL: %v", ca.sql)) + require.EqualValues(t, ca.isLeaders, hotRegionsHistoryExtractor.IsLeaders, "SQL: %v", ca.sql) } if ca.hotRegionTypes.Count() > 0 { - c.Assert(hotRegionsHistoryExtractor.HotRegionTypes, DeepEquals, ca.hotRegionTypes, Commentf("SQL: %v", ca.sql)) + require.EqualValues(t, ca.hotRegionTypes, hotRegionsHistoryExtractor.HotRegionTypes, "SQL: %v", ca.sql) } if len(ca.regionIDs) > 0 { - c.Assert(hotRegionsHistoryExtractor.RegionIDs, DeepEquals, ca.regionIDs, Commentf("SQL: %v", ca.sql)) + require.EqualValues(t, ca.regionIDs, hotRegionsHistoryExtractor.RegionIDs, "SQL: %v", ca.sql) } if len(ca.storeIDs) > 0 { - c.Assert(hotRegionsHistoryExtractor.StoreIDs, DeepEquals, ca.storeIDs, Commentf("SQL: %v", ca.sql)) + require.EqualValues(t, ca.storeIDs, hotRegionsHistoryExtractor.StoreIDs, "SQL: %v", ca.sql) } if len(ca.peerIDs) > 0 { - c.Assert(hotRegionsHistoryExtractor.PeerIDs, DeepEquals, ca.peerIDs, Commentf("SQL: %v", ca.sql)) + require.EqualValues(t, ca.peerIDs, hotRegionsHistoryExtractor.PeerIDs, "SQL: %v", ca.sql) } } } + +func TestPredicateQuery(t *testing.T) { + store, clean := testkit.CreateMockStore(t) + defer clean() + + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("create table t(id int, abclmn int);") + + tk.MustQuery("show columns from t like 'abclmn'").Check(testutil.RowsWithSep(",", "abclmn,int(11),YES,,,")) + tk.MustQuery("show columns from t like 'ABCLMN'").Check(testutil.RowsWithSep(",", "abclmn,int(11),YES,,,")) + tk.MustQuery("show columns from t like 'abc%'").Check(testutil.RowsWithSep(",", "abclmn,int(11),YES,,,")) + tk.MustQuery("show columns from t like 'ABC%'").Check(testutil.RowsWithSep(",", "abclmn,int(11),YES,,,")) + tk.MustQuery("show columns from t like '%lmn'").Check(testutil.RowsWithSep(",", "abclmn,int(11),YES,,,")) + tk.MustQuery("show columns from t like '%LMN'").Check(testutil.RowsWithSep(",", "abclmn,int(11),YES,,,")) + tk.MustQuery("show columns in t like '%lmn'").Check(testutil.RowsWithSep(",", "abclmn,int(11),YES,,,")) + tk.MustQuery("show columns in t like '%LMN'").Check(testutil.RowsWithSep(",", "abclmn,int(11),YES,,,")) + tk.MustQuery("show fields in t like '%lmn'").Check(testutil.RowsWithSep(",", "abclmn,int(11),YES,,,")) + tk.MustQuery("show fields in t like '%LMN'").Check(testutil.RowsWithSep(",", "abclmn,int(11),YES,,,")) + + tk.MustQuery("show columns from t where field like '%lmn'").Check(testutil.RowsWithSep(",", "abclmn,int(11),YES,,,")) + tk.MustQuery("show columns from t where field = 'abclmn'").Check(testutil.RowsWithSep(",", "abclmn,int(11),YES,,,")) + tk.MustQuery("show columns in t where field = 'abclmn'").Check(testutil.RowsWithSep(",", "abclmn,int(11),YES,,,")) + tk.MustQuery("show fields from t where field = 'abclmn'").Check(testutil.RowsWithSep(",", "abclmn,int(11),YES,,,")) + tk.MustQuery("show fields in t where field = 'abclmn'").Check(testutil.RowsWithSep(",", "abclmn,int(11),YES,,,")) + tk.MustQuery("explain t").Check(testkit.Rows("id int(11) YES ", "abclmn int(11) YES ")) + + tk.MustGetErrCode("show columns from t like id", errno.ErrBadField) + tk.MustGetErrCode("show columns from t like `id`", errno.ErrBadField) +} diff --git a/planner/core/physical_plans.go b/planner/core/physical_plans.go index 6293bba4b5073..876e71415b75e 100644 --- a/planner/core/physical_plans.go +++ b/planner/core/physical_plans.go @@ -1367,6 +1367,8 @@ type PhysicalShow struct { physicalSchemaProducer ShowContents + + Extractor ShowPredicateExtractor } // PhysicalShowDDLJobs is for showing DDL job list. diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index 0dfd7ea71c743..026162be967d6 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -2871,7 +2871,15 @@ func (b *PlanBuilder) buildShow(ctx context.Context, show *ast.ShowStmt) (Plan, }.Init(b.ctx) isView := false isSequence := false + switch show.Tp { + case ast.ShowColumns: + var extractor ShowColumnsTableExtractor + if extractor.Extract(show) { + p.Extractor = &extractor + // avoid to build Selection. + show.Pattern = nil + } case ast.ShowTables, ast.ShowTableStatus: if p.DBName == "" { return nil, ErrNoDB diff --git a/planner/core/show_predicate_extractor.go b/planner/core/show_predicate_extractor.go new file mode 100644 index 0000000000000..0be76a053fd6d --- /dev/null +++ b/planner/core/show_predicate_extractor.go @@ -0,0 +1,97 @@ +// Copyright 2022 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package core + +import ( + "bytes" + "fmt" + "strings" + + "github.com/pingcap/tidb/parser/ast" + driver "github.com/pingcap/tidb/types/parser_driver" + "github.com/pingcap/tidb/util/collate" + "github.com/pingcap/tidb/util/stringutil" +) + +var _ ShowPredicateExtractor = &ShowColumnsTableExtractor{} + +// ShowPredicateExtractor is used to extract some predicates from `PatternLikeExpr` clause +// and push the predicates down to the data retrieving on reading memory table stage when use ShowStmt. +// +// e.g: +// SHOW COLUMNS FROM t LIKE '%abc%' +// We must request all components from the memory table, and filter the result by the PatternLikeExpr predicate. +// +// it is a way to fix https://github.com/pingcap/tidb/issues/29910. +type ShowPredicateExtractor interface { + // Extracts predicates which can be pushed down and returns whether the extractor can extract predicates. + Extract(show *ast.ShowStmt) bool + explainInfo() string +} + +// ShowColumnsTableExtractor is used to extract some predicates of tables table. +type ShowColumnsTableExtractor struct { + Field string + + FieldPatterns string +} + +// Extract implements the MemTablePredicateExtractor Extract interface +func (e *ShowColumnsTableExtractor) Extract(show *ast.ShowStmt) bool { + if show.Pattern != nil && show.Pattern.Pattern != nil { + pattern := show.Pattern + switch pattern.Pattern.(type) { + case *driver.ValueExpr: + // It is used in `SHOW COLUMNS FROM t LIKE `abc``. + ptn := pattern.Pattern.(*driver.ValueExpr).GetString() + patValue, patTypes := stringutil.CompilePattern(ptn, pattern.Escape) + if !collate.NewCollationEnabled() && stringutil.IsExactMatch(patTypes) { + e.Field = strings.ToLower(string(patValue)) + return true + } + // (?i) mean to be case-insensitive. + e.FieldPatterns = "(?i)" + stringutil.CompileLike2Regexp(string(patValue)) + return true + case *ast.ColumnNameExpr: + // It is used in `SHOW COLUMNS FROM t LIKE abc`. + // MySQL do not support this syntax and return the error. + return false + } + } else if show.Column != nil && show.Column.Name.L != "" { + // it is used in `DESCRIBE t COLUMN`. + e.Field = show.Column.Name.L + return true + } + return false + +} + +func (e *ShowColumnsTableExtractor) explainInfo() string { + r := new(bytes.Buffer) + if len(e.Field) > 0 { + r.WriteString(fmt.Sprintf("field:[%s], ", e.Field)) + } + + if len(e.FieldPatterns) > 0 { + r.WriteString(fmt.Sprintf("field_pattern:[%s], ", e.FieldPatterns)) + } + + // remove the last ", " in the message info + s := r.String() + if len(s) > 2 { + return s[:len(s)-2] + } + return s +} diff --git a/planner/core/stringer.go b/planner/core/stringer.go index ce41a53bf66c8..68eb94c584861 100644 --- a/planner/core/stringer.go +++ b/planner/core/stringer.go @@ -132,8 +132,16 @@ func toString(in Plan, strs []string, idxs []int) ([]string, []int) { str = "Lock" case *ShowDDL: str = "ShowDDL" - case *LogicalShow, *PhysicalShow: + case *LogicalShow: str = "Show" + if pl := in.(*LogicalShow); pl.Extractor != nil { + str = str + "(" + pl.Extractor.explainInfo() + ")" + } + case *PhysicalShow: + str = "Show" + if pl := in.(*PhysicalShow); pl.Extractor != nil { + str = str + "(" + pl.Extractor.explainInfo() + ")" + } case *LogicalShowDDLJobs, *PhysicalShowDDLJobs: str = "ShowDDLJobs" case *LogicalSort, *PhysicalSort: diff --git a/planner/core/stringer_test.go b/planner/core/stringer_test.go new file mode 100644 index 0000000000000..3ca5cbfc66568 --- /dev/null +++ b/planner/core/stringer_test.go @@ -0,0 +1,72 @@ +// Copyright 2022 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package core_test + +import ( + "context" + "testing" + + "github.com/pingcap/tidb/parser" + "github.com/pingcap/tidb/planner/core" + "github.com/pingcap/tidb/testkit" + "github.com/pingcap/tidb/util/hint" + "github.com/stretchr/testify/require" +) + +func TestPlanStringer(t *testing.T) { + store, clean := testkit.CreateMockStore(t) + defer clean() + tk := testkit.NewTestKit(t, store) + + tk.MustExec("use test") + tk.MustExec("create table t(a int, b int, c int, index idx(a))") + tests := []struct { + sql string + plan string + }{ + { + sql: "show columns from t like 'a'", + plan: "Show(field:[a])", + }, + { + sql: "show columns from t like 'a%'", + plan: "Show(field_pattern:[(?i)a.*])", + }, + { + sql: "show columns from t where field = 'a'", + plan: "Show->Sel([eq(Column#13, a)])->Projection", + }, + { + sql: "desc t", + plan: "Show", + }, + { + sql: "desc t a", + plan: "Show(field:[a])", + }, + } + parser := parser.New() + for _, tt := range tests { + stmt, err := parser.ParseOneStmt(tt.sql, "", "") + require.NoError(t, err, "for %s", tt.sql) + ret := &core.PreprocessorReturn{} + builder, _ := core.NewPlanBuilder().Init(tk.Session(), ret.InfoSchema, &hint.BlockHintProcessor{}) + p, err := builder.Build(context.TODO(), stmt) + require.NoError(t, err, "for %s", tt.sql) + p, err = core.LogicalOptimize(context.TODO(), builder.GetOptFlag(), p.(core.LogicalPlan)) + require.NoError(t, err, "for %s", tt.sql) + require.Equal(t, tt.plan, core.ToString(p)) + } +} From b1c5a67fe7eaf4b399792cd158681c543a864eff Mon Sep 17 00:00:00 2001 From: ti-srebot <66930949+ti-srebot@users.noreply.github.com> Date: Thu, 31 Mar 2022 11:46:29 +0800 Subject: [PATCH 3/7] *: add show push down for ShowTables (#31919) (#33338) close pingcap/tidb#30803 --- executor/show.go | 24 +++++++-- .../core/memtable_predicate_extractor_test.go | 11 +++- planner/core/planbuilder.go | 12 ++++- planner/core/show_predicate_extractor.go | 50 ++++++++++++++++++- planner/core/stringer_test.go | 16 ++++++ 5 files changed, 106 insertions(+), 7 deletions(-) diff --git a/executor/show.go b/executor/show.go index 5116ea81056e8..36d559c706173 100644 --- a/executor/show.go +++ b/executor/show.go @@ -431,14 +431,32 @@ func (e *ShowExec) fetchShowTables() error { return ErrBadDB.GenWithStackByArgs(e.DBName) } // sort for tables - tableNames := make([]string, 0, len(e.is.SchemaTables(e.DBName))) + schemaTables := e.is.SchemaTables(e.DBName) + tableNames := make([]string, 0, len(schemaTables)) activeRoles := e.ctx.GetSessionVars().ActiveRoles - var tableTypes = make(map[string]string) - for _, v := range e.is.SchemaTables(e.DBName) { + var ( + tableTypes = make(map[string]string) + fieldPatternsRegexp *regexp.Regexp + FieldFilterEnable bool + fieldFilter string + ) + if e.Extractor != nil { + extractor := (e.Extractor).(*plannercore.ShowTablesTableExtractor) + if extractor.FieldPatterns != "" { + fieldPatternsRegexp = regexp.MustCompile(extractor.FieldPatterns) + } + FieldFilterEnable = extractor.Field != "" + fieldFilter = extractor.Field + } + for _, v := range schemaTables { // Test with mysql.AllPrivMask means any privilege would be OK. // TODO: Should consider column privileges, which also make a table visible. if checker != nil && !checker.RequestVerification(activeRoles, e.DBName.O, v.Meta().Name.O, "", mysql.AllPrivMask) { continue + } else if FieldFilterEnable && v.Meta().Name.L != fieldFilter { + continue + } else if fieldPatternsRegexp != nil && !fieldPatternsRegexp.MatchString(v.Meta().Name.L) { + continue } tableNames = append(tableNames, v.Meta().Name.O) if v.Meta().IsView() { diff --git a/planner/core/memtable_predicate_extractor_test.go b/planner/core/memtable_predicate_extractor_test.go index 0fc8c4abf3743..761b87bcfd4ab 100644 --- a/planner/core/memtable_predicate_extractor_test.go +++ b/planner/core/memtable_predicate_extractor_test.go @@ -1427,7 +1427,7 @@ func TestPredicateQuery(t *testing.T) { tk := testkit.NewTestKit(t, store) tk.MustExec("use test") tk.MustExec("create table t(id int, abclmn int);") - + tk.MustExec("create table abclmn(a int);") tk.MustQuery("show columns from t like 'abclmn'").Check(testutil.RowsWithSep(",", "abclmn,int(11),YES,,,")) tk.MustQuery("show columns from t like 'ABCLMN'").Check(testutil.RowsWithSep(",", "abclmn,int(11),YES,,,")) tk.MustQuery("show columns from t like 'abc%'").Check(testutil.RowsWithSep(",", "abclmn,int(11),YES,,,")) @@ -1448,4 +1448,13 @@ func TestPredicateQuery(t *testing.T) { tk.MustGetErrCode("show columns from t like id", errno.ErrBadField) tk.MustGetErrCode("show columns from t like `id`", errno.ErrBadField) + + tk.MustQuery("show tables like 't'").Check(testkit.Rows("t")) + tk.MustQuery("show tables like 'T'").Check(testkit.Rows("t")) + tk.MustQuery("show tables like 'ABCLMN'").Check(testkit.Rows("abclmn")) + tk.MustQuery("show tables like 'ABC%'").Check(testkit.Rows("abclmn")) + tk.MustQuery("show tables like '%lmn'").Check(testkit.Rows("abclmn")) + tk.MustQuery("show full tables like '%lmn'").Check(testkit.Rows("abclmn BASE TABLE")) + tk.MustGetErrCode("show tables like T", errno.ErrBadField) + tk.MustGetErrCode("show tables like `T`", errno.ErrBadField) } diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index 026162be967d6..131cbe9c35466 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -2880,7 +2880,17 @@ func (b *PlanBuilder) buildShow(ctx context.Context, show *ast.ShowStmt) (Plan, // avoid to build Selection. show.Pattern = nil } - case ast.ShowTables, ast.ShowTableStatus: + case ast.ShowTables: + if p.DBName == "" { + return nil, ErrNoDB + } + var extractor ShowTablesTableExtractor + if extractor.Extract(show) { + p.Extractor = &extractor + // Avoid building Selection. + show.Pattern = nil + } + case ast.ShowTableStatus: if p.DBName == "" { return nil, ErrNoDB } diff --git a/planner/core/show_predicate_extractor.go b/planner/core/show_predicate_extractor.go index 0be76a053fd6d..103c4107c7f5e 100644 --- a/planner/core/show_predicate_extractor.go +++ b/planner/core/show_predicate_extractor.go @@ -25,7 +25,10 @@ import ( "github.com/pingcap/tidb/util/stringutil" ) -var _ ShowPredicateExtractor = &ShowColumnsTableExtractor{} +var ( + _ ShowPredicateExtractor = &ShowColumnsTableExtractor{} + _ ShowPredicateExtractor = &ShowTablesTableExtractor{} +) // ShowPredicateExtractor is used to extract some predicates from `PatternLikeExpr` clause // and push the predicates down to the data retrieving on reading memory table stage when use ShowStmt. @@ -75,7 +78,6 @@ func (e *ShowColumnsTableExtractor) Extract(show *ast.ShowStmt) bool { return true } return false - } func (e *ShowColumnsTableExtractor) explainInfo() string { @@ -95,3 +97,47 @@ func (e *ShowColumnsTableExtractor) explainInfo() string { } return s } + +// ShowTablesTableExtractor is used to extract some predicates of tables. +type ShowTablesTableExtractor struct { + ShowColumnsTableExtractor +} + +// Extract implements the ShowTablesTableExtractor Extract interface +func (e *ShowTablesTableExtractor) Extract(show *ast.ShowStmt) bool { + if show.Pattern != nil && show.Pattern.Pattern != nil { + pattern := show.Pattern + switch pattern.Pattern.(type) { + case *driver.ValueExpr: + // It is used in `SHOW TABLE in t LIKE `abc``. + ptn := pattern.Pattern.(*driver.ValueExpr).GetString() + patValue, patTypes := stringutil.CompilePattern(ptn, pattern.Escape) + if stringutil.IsExactMatch(patTypes) { + e.Field = strings.ToLower(string(patValue)) + return true + } + // (?i) mean to be case-insensitive. + e.FieldPatterns = "(?i)" + stringutil.CompileLike2Regexp(string(patValue)) + return true + } + } + return false +} + +func (e *ShowTablesTableExtractor) explainInfo() string { + r := new(bytes.Buffer) + if len(e.Field) > 0 { + r.WriteString(fmt.Sprintf("table:[%s], ", e.Field)) + } + + if len(e.FieldPatterns) > 0 { + r.WriteString(fmt.Sprintf("table_pattern:[%s], ", e.FieldPatterns)) + } + + // remove the last ", " in the message info + s := r.String() + if len(s) > 2 { + return s[:len(s)-2] + } + return s +} diff --git a/planner/core/stringer_test.go b/planner/core/stringer_test.go index 3ca5cbfc66568..e4163c7c29c8e 100644 --- a/planner/core/stringer_test.go +++ b/planner/core/stringer_test.go @@ -56,6 +56,22 @@ func TestPlanStringer(t *testing.T) { sql: "desc t a", plan: "Show(field:[a])", }, + { + sql: "show tables in test like 't'", + plan: "Show(table:[t])", + }, + { + sql: "show tables in test like 'T'", + plan: "Show(table:[t])", + }, + { + sql: "show tables in test like 't%'", + plan: "Show(table_pattern:[(?i)t.*])", + }, + { + sql: "show tables in test like '%T%'", + plan: "Show(table_pattern:[(?i).*T.*])", + }, } parser := parser.New() for _, tt := range tests { From 8b867ed4de59ed26af05bf8479194fba31847dad Mon Sep 17 00:00:00 2001 From: ti-srebot <66930949+ti-srebot@users.noreply.github.com> Date: Mon, 18 Apr 2022 14:18:03 +0800 Subject: [PATCH 4/7] stats: resolve the constant selectivity when handling DNF exprs (#31242) (#31685) close pingcap/tidb#31096 --- planner/core/integration_test.go | 38 +++++++++++++++++++ .../core/testdata/integration_suite_out.json | 4 +- statistics/selectivity.go | 20 +++++++++- 3 files changed, 59 insertions(+), 3 deletions(-) diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index 769e9782ace39..7e1c9c1cf9612 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -5226,3 +5226,41 @@ func (s *testIntegrationSuite) TestIssue31202(c *C) { "└─TableFullScan 10000.00 cop[tikv] table:t31202 keep order:false, stats:pseudo")) tk.MustExec("drop table if exists t31202") } + +// TestDNFCondSelectivityWithConst test selectivity calculation with DNF conditions with one is const. +// Close https://github.com/pingcap/tidb/issues/31096 +func (s *testIntegrationSuite) TestDNFCondSelectivityWithConst(c *C) { + testKit := testkit.NewTestKit(c, s.store) + testKit.MustExec("use test") + testKit.MustExec("drop table if exists t1") + testKit.MustExec("create table t1(a int, b int, c int);") + testKit.MustExec("insert into t1 value(10,10,10)") + for i := 0; i < 7; i++ { + testKit.MustExec("insert into t1 select * from t1") + } + testKit.MustExec("insert into t1 value(1,1,1)") + testKit.MustExec("analyze table t1") + + testKit.MustQuery("explain select * from t1 where a=1 or b=1;").Check(testkit.Rows("TableReader_7 1.99 root data:Selection_6]\n" + + "[└─Selection_6 1.99 cop[tikv] or(eq(test.t1.a, 1), eq(test.t1.b, 1))]\n" + + "[ └─TableFullScan_5 129.00 cop[tikv] table:t1 keep order:false")) + testKit.MustQuery("explain select * from t1 where 0=1 or a=1 or b=1;").Check(testkit.Rows("TableReader_7 1.99 root data:Selection_6]\n" + + "[└─Selection_6 1.99 cop[tikv] or(0, or(eq(test.t1.a, 1), eq(test.t1.b, 1)))]\n" + + "[ └─TableFullScan_5 129.00 cop[tikv] table:t1 keep order:false")) + testKit.MustQuery("explain select * from t1 where null or a=1 or b=1;").Check(testkit.Rows("TableReader_7 1.99 root data:Selection_6]\n" + + "[└─Selection_6 1.99 cop[tikv] or(0, or(eq(test.t1.a, 1), eq(test.t1.b, 1)))]\n" + + "[ └─TableFullScan_5 129.00 cop[tikv] table:t1 keep order:false")) + testKit.MustQuery("explain select * from t1 where a=1 or false or b=1;").Check(testkit.Rows("TableReader_7 1.99 root data:Selection_6]\n" + + "[└─Selection_6 1.99 cop[tikv] or(eq(test.t1.a, 1), or(0, eq(test.t1.b, 1)))]\n" + + "[ └─TableFullScan_5 129.00 cop[tikv] table:t1 keep order:false")) + testKit.MustQuery("explain select * from t1 where a=1 or b=1 or \"false\";").Check(testkit.Rows("TableReader_7 1.99 root data:Selection_6]\n" + + "[└─Selection_6 1.99 cop[tikv] or(eq(test.t1.a, 1), or(eq(test.t1.b, 1), 0))]\n" + + "[ └─TableFullScan_5 129.00 cop[tikv] table:t1 keep order:false")) + testKit.MustQuery("explain select * from t1 where 1=1 or a=1 or b=1;").Check(testkit.Rows("TableReader_7 129.00 root data:Selection_6]\n" + + "[└─Selection_6 129.00 cop[tikv] or(1, or(eq(test.t1.a, 1), eq(test.t1.b, 1)))]\n" + + "[ └─TableFullScan_5 129.00 cop[tikv] table:t1 keep order:false")) + testKit.MustQuery("explain select * from t1 where a=1 or b=1 or 1=1;").Check(testkit.Rows("TableReader_7 129.00 root data:Selection_6]\n" + + "[└─Selection_6 129.00 cop[tikv] or(eq(test.t1.a, 1), or(eq(test.t1.b, 1), 1))]\n" + + "[ └─TableFullScan_5 129.00 cop[tikv] table:t1 keep order:false")) + testKit.MustExec("drop table if exists t1") +} diff --git a/planner/core/testdata/integration_suite_out.json b/planner/core/testdata/integration_suite_out.json index 5efa0586a2cd9..dd4330fe1310e 100644 --- a/planner/core/testdata/integration_suite_out.json +++ b/planner/core/testdata/integration_suite_out.json @@ -355,8 +355,8 @@ { "SQL": "explain format = 'brief' select /*+ USE_INDEX_MERGE(t, a, b, c) */ * from t where 1 or t.a = 1 or t.b = 2", "Plan": [ - "TableReader 8000.40 root data:Selection", - "└─Selection 8000.40 cop[tikv] or(1, or(eq(test.t.a, 1), eq(test.t.b, 2)))", + "TableReader 10000.00 root data:Selection", + "└─Selection 10000.00 cop[tikv] or(1, or(eq(test.t.a, 1), eq(test.t.b, 2)))", " └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" ] } diff --git a/statistics/selectivity.go b/statistics/selectivity.go index 45db1cebf9b1c..0f5f4792acc71 100644 --- a/statistics/selectivity.go +++ b/statistics/selectivity.go @@ -354,6 +354,24 @@ func (coll *HistColl) Selectivity(ctx sessionctx.Context, exprs []expression.Exp if ok { continue } + // where {"0" / 0 / "false" / false / null} or A or B ... the '0' constant item should be ignored. + if c, ok := cond.(*expression.Constant); ok { + if !expression.MaybeOverOptimized4PlanCache(ctx, []expression.Expression{cond}) { + if c.Value.IsNull() { + // constant is null + continue + } + if isTrue, err := c.Value.ToBool(sc); err == nil { + if isTrue == 0 { + // constant == 0 + continue + } + // constant == 1 + selectivity = 1.0 + break + } + } + } var cnfItems []expression.Expression if scalar, ok := cond.(*expression.ScalarFunction); ok && scalar.FuncName.L == ast.LogicAnd { @@ -365,7 +383,7 @@ func (coll *HistColl) Selectivity(ctx sessionctx.Context, exprs []expression.Exp curSelectivity, _, err := coll.Selectivity(ctx, cnfItems, nil) if err != nil { logutil.BgLogger().Debug("something wrong happened, use the default selectivity", zap.Error(err)) - selectivity = selectionFactor + curSelectivity = selectionFactor } selectivity = selectivity + curSelectivity - selectivity*curSelectivity From e5e8faea8ff6b80448b46067d68a6f2affcbf1e1 Mon Sep 17 00:00:00 2001 From: ti-srebot <66930949+ti-srebot@users.noreply.github.com> Date: Wed, 20 Apr 2022 17:42:04 +0800 Subject: [PATCH 5/7] planner: support MaxOneRow clone (#33888) (#34067) close pingcap/tidb#33887 --- executor/join.go | 1 + planner/core/optimizer.go | 2 ++ planner/core/physical_plans.go | 11 +++++++++++ planner/core/planbuilder_test.go | 5 +++++ 4 files changed, 19 insertions(+) diff --git a/executor/join.go b/executor/join.go index 09990e6c5d4b1..702a5cf23632c 100644 --- a/executor/join.go +++ b/executor/join.go @@ -855,6 +855,7 @@ func (e *NestedLoopApplyExec) Close() error { } else { runtimeStats.setCacheInfo(false, 0) } + runtimeStats.SetConcurrencyInfo(execdetails.NewConcurrencyInfo("Concurrency", 0)) } return e.outerExec.Close() } diff --git a/planner/core/optimizer.go b/planner/core/optimizer.go index 5cdd693a7e77e..0b605f3eecd95 100644 --- a/planner/core/optimizer.go +++ b/planner/core/optimizer.go @@ -392,6 +392,8 @@ func enableParallelApply(sctx sessionctx.Context, plan PhysicalPlan) PhysicalPla supportClone := err == nil // limitation 2 if noOrder && supportClone { apply.Concurrency = sctx.GetSessionVars().ExecutorConcurrency + } else { + sctx.GetSessionVars().StmtCtx.AppendWarning(errors.Errorf("Some apply operators can not be executed in parallel")) } // because of the limitation 3, we cannot parallelize Apply operators in this Apply's inner size, diff --git a/planner/core/physical_plans.go b/planner/core/physical_plans.go index 876e71415b75e..3d96dd71de13a 100644 --- a/planner/core/physical_plans.go +++ b/planner/core/physical_plans.go @@ -1245,6 +1245,17 @@ type PhysicalMaxOneRow struct { basePhysicalPlan } +// Clone implements PhysicalPlan interface. +func (p *PhysicalMaxOneRow) Clone() (PhysicalPlan, error) { + cloned := new(PhysicalMaxOneRow) + base, err := p.basePhysicalPlan.cloneWithSelf(cloned) + if err != nil { + return nil, err + } + cloned.basePhysicalPlan = *base + return cloned, nil +} + // PhysicalTableDual is the physical operator of dual. type PhysicalTableDual struct { physicalSchemaProducer diff --git a/planner/core/planbuilder_test.go b/planner/core/planbuilder_test.go index 05f8bf37e5838..5eb5bd59948eb 100644 --- a/planner/core/planbuilder_test.go +++ b/planner/core/planbuilder_test.go @@ -287,6 +287,11 @@ func (s *testPlanBuilderSuite) TestPhysicalPlanClone(c *C) { sel = sel.Init(ctx, stats, 0) c.Assert(checkPhysicalPlanClone(sel), IsNil) + // maxOneRow + maxOneRow := &PhysicalMaxOneRow{} + maxOneRow = maxOneRow.Init(ctx, stats, 0) + c.Assert(checkPhysicalPlanClone(maxOneRow), IsNil) + // projection proj := &PhysicalProjection{Exprs: []expression.Expression{col, cst}} proj = proj.Init(ctx, stats, 0) From c9b917b5de42c29083e632b291099a49fa3286f8 Mon Sep 17 00:00:00 2001 From: ti-srebot <66930949+ti-srebot@users.noreply.github.com> Date: Thu, 21 Apr 2022 12:52:03 +0800 Subject: [PATCH 6/7] lightning: add retry and early terminate checking empty table (#31798) (#33148) close pingcap/tidb#31797 --- br/pkg/lightning/restore/check_info.go | 24 ++++++++++++++++----- br/pkg/lightning/restore/check_info_test.go | 16 ++++++++++++++ br/pkg/lightning/restore/meta_manager.go | 2 +- br/pkg/lightning/restore/restore.go | 4 ++-- br/pkg/lightning/restore/table_restore.go | 4 ++-- 5 files changed, 40 insertions(+), 10 deletions(-) diff --git a/br/pkg/lightning/restore/check_info.go b/br/pkg/lightning/restore/check_info.go index 70f424bae5f16..b5ba12cc40287 100644 --- a/br/pkg/lightning/restore/check_info.go +++ b/br/pkg/lightning/restore/check_info.go @@ -1097,6 +1097,7 @@ func (rc *Controller) checkTableEmpty(ctx context.Context) error { concurrency := utils.MinInt(tableCount, rc.cfg.App.RegionConcurrency) ch := make(chan string, concurrency) eg, gCtx := errgroup.WithContext(ctx) + for i := 0; i < concurrency; i++ { eg.Go(func() error { for tblName := range ch { @@ -1125,9 +1126,15 @@ func (rc *Controller) checkTableEmpty(ctx context.Context) error { return nil }) } +loop: for _, db := range rc.dbMetas { for _, tbl := range db.Tables { - ch <- common.UniqueTable(tbl.DB, tbl.Name) + select { + case ch <- common.UniqueTable(tbl.DB, tbl.Name): + case <-gCtx.Done(): + break loop + } + } } close(ch) @@ -1135,7 +1142,7 @@ func (rc *Controller) checkTableEmpty(ctx context.Context) error { if common.IsContextCanceledError(err) { return nil } - return errors.Trace(err) + return errors.Annotate(err, "check table contains data failed") } if len(tableNames) > 0 { @@ -1147,13 +1154,20 @@ func (rc *Controller) checkTableEmpty(ctx context.Context) error { return nil } -func tableContainsData(ctx context.Context, db utils.QueryExecutor, tableName string) (bool, error) { +func tableContainsData(ctx context.Context, db utils.DBExecutor, tableName string) (bool, error) { + failpoint.Inject("CheckTableEmptyFailed", func() { + failpoint.Return(false, errors.New("mock error")) + }) query := "select 1 from " + tableName + " limit 1" + exec := common.SQLWithRetry{ + DB: db, + Logger: log.L(), + } var dump int - err := db.QueryRowContext(ctx, query).Scan(&dump) + err := exec.QueryRow(ctx, "check table empty", query, &dump) switch { - case err == sql.ErrNoRows: + case errors.ErrorEqual(err, sql.ErrNoRows): return false, nil case err != nil: return false, errors.Trace(err) diff --git a/br/pkg/lightning/restore/check_info_test.go b/br/pkg/lightning/restore/check_info_test.go index c679298f6a612..fd73cc6504e82 100644 --- a/br/pkg/lightning/restore/check_info_test.go +++ b/br/pkg/lightning/restore/check_info_test.go @@ -32,6 +32,7 @@ import ( "github.com/pingcap/tidb/br/pkg/lightning/worker" "github.com/pingcap/tidb/br/pkg/storage" "github.com/pingcap/tidb/ddl" + "github.com/pingcap/tidb/errno" "github.com/pingcap/tidb/parser" "github.com/pingcap/tidb/parser/ast" "github.com/pingcap/tidb/parser/model" @@ -476,6 +477,9 @@ func (s *checkInfoSuite) TestCheckTableEmpty(c *C) { c.Assert(err, IsNil) rc.tidbGlue = glue.NewExternalTiDBGlue(db, mysql.ModeNone) mock.MatchExpectationsInOrder(false) + // test auto retry retryable error + mock.ExpectQuery("select 1 from `test1`.`tbl1` limit 1"). + WillReturnError(mysql.NewErr(errno.ErrPDServerTimeout)) mock.ExpectQuery("select 1 from `test1`.`tbl1` limit 1"). WillReturnRows(sqlmock.NewRows([]string{""}).RowError(0, sql.ErrNoRows)) mock.ExpectQuery("select 1 from `test1`.`tbl2` limit 1"). @@ -541,6 +545,18 @@ func (s *checkInfoSuite) TestCheckTableEmpty(c *C) { err = rc.checkTableEmpty(ctx) c.Assert(err, IsNil) c.Assert(mock.ExpectationsWereMet(), IsNil) + + err = failpoint.Enable("github.com/pingcap/tidb/br/pkg/lightning/restore/CheckTableEmptyFailed", `return`) + c.Assert(err, IsNil) + defer func() { + _ = failpoint.Disable("github.com/pingcap/tidb/br/pkg/lightning/restore/CheckTableEmptyFailed") + }() + + // restrict the concurrency to ensure there are more tables than workers + rc.cfg.App.RegionConcurrency = 1 + // test check tables not stuck but return the right error + err = rc.checkTableEmpty(ctx) + c.Assert(err, ErrorMatches, ".*check table contains data failed: mock error.*") } func (s *checkInfoSuite) TestLocalResource(c *C) { diff --git a/br/pkg/lightning/restore/meta_manager.go b/br/pkg/lightning/restore/meta_manager.go index 58d8c59966a6b..a8b43167c83ea 100644 --- a/br/pkg/lightning/restore/meta_manager.go +++ b/br/pkg/lightning/restore/meta_manager.go @@ -206,7 +206,7 @@ func (m *dbTableMetaMgr) AllocTableRowIDs(ctx context.Context, rawRowIDMax int64 } if status == metaStatusChecksuming { - return errors.New("target table is calculating checksum, please wait unit the checksum is finished and try again.") + return errors.New("Target table is calculating checksum. Please wait until the checksum is finished and try again.") } if metaTaskID == m.taskID { diff --git a/br/pkg/lightning/restore/restore.go b/br/pkg/lightning/restore/restore.go index 4dd6cc80c8480..75cb9b0b6729d 100644 --- a/br/pkg/lightning/restore/restore.go +++ b/br/pkg/lightning/restore/restore.go @@ -1680,12 +1680,10 @@ func (rc *Controller) doCompact(ctx context.Context, level int32) error { } func (rc *Controller) switchToImportMode(ctx context.Context) { - log.L().Info("switch to import mode") rc.switchTiKVMode(ctx, sstpb.SwitchMode_Import) } func (rc *Controller) switchToNormalMode(ctx context.Context) { - log.L().Info("switch to normal mode") rc.switchTiKVMode(ctx, sstpb.SwitchMode_Normal) } @@ -1695,6 +1693,8 @@ func (rc *Controller) switchTiKVMode(ctx context.Context, mode sstpb.SwitchMode) return } + log.L().Info("switch import mode", zap.Stringer("mode", mode)) + // It is fine if we miss some stores which did not switch to Import mode, // since we're running it periodically, so we exclude disconnected stores. // But it is essential all stores be switched back to Normal mode to allow diff --git a/br/pkg/lightning/restore/table_restore.go b/br/pkg/lightning/restore/table_restore.go index e340063d8a34b..6fe9a22dcc431 100644 --- a/br/pkg/lightning/restore/table_restore.go +++ b/br/pkg/lightning/restore/table_restore.go @@ -711,8 +711,8 @@ func (tr *TableRestore) postProcess( } // tidb backend don't need checksum & analyze - if !rc.backend.ShouldPostProcess() { - tr.logger.Debug("skip checksum & analyze, not supported by this backend") + if rc.cfg.PostRestore.Checksum == config.OpLevelOff && rc.cfg.PostRestore.Analyze == config.OpLevelOff { + tr.logger.Debug("skip checksum & analyze, either because not supported by this backend or manually disabled") err := rc.saveStatusCheckpoint(ctx, tr.tableName, checkpoints.WholeTableEngineID, nil, checkpoints.CheckpointStatusAnalyzeSkipped) return false, errors.Trace(err) } From 52c636cf1749a587cda0980a731386a2c6a98d7c Mon Sep 17 00:00:00 2001 From: ti-srebot <66930949+ti-srebot@users.noreply.github.com> Date: Thu, 21 Apr 2022 14:16:03 +0800 Subject: [PATCH 7/7] lightning: support charset for create schema (#31915) (#32258) close pingcap/tidb#31913 --- br/pkg/lightning/mydump/loader.go | 50 +++++++++++--- br/pkg/lightning/mydump/loader_test.go | 29 +++++--- br/pkg/lightning/restore/restore.go | 69 +++++++------------ br/pkg/lightning/restore/tidb.go | 14 ++-- br/pkg/lightning/restore/tidb_test.go | 42 ++++++----- .../data/perm-schema-create.sql | 2 +- .../data/nc-schema-create.sql | 1 + .../data/nc.ci-schema.sql | 1 + .../lightning_new_collation/data/nc.ci.0.csv | 2 + br/tests/lightning_new_collation/run.sh | 4 ++ 10 files changed, 124 insertions(+), 90 deletions(-) create mode 100644 br/tests/lightning_new_collation/data/nc-schema-create.sql create mode 100644 br/tests/lightning_new_collation/data/nc.ci-schema.sql create mode 100644 br/tests/lightning_new_collation/data/nc.ci.0.csv diff --git a/br/pkg/lightning/mydump/loader.go b/br/pkg/lightning/mydump/loader.go index 27bab8fa5cf7b..e50f61c1dce5a 100644 --- a/br/pkg/lightning/mydump/loader.go +++ b/br/pkg/lightning/mydump/loader.go @@ -18,10 +18,12 @@ import ( "context" "path/filepath" "sort" + "strings" "github.com/pingcap/errors" filter "github.com/pingcap/tidb-tools/pkg/table-filter" router "github.com/pingcap/tidb-tools/pkg/table-router" + "github.com/pingcap/tidb/br/pkg/lightning/common" "github.com/pingcap/tidb/br/pkg/lightning/config" "github.com/pingcap/tidb/br/pkg/lightning/log" "github.com/pingcap/tidb/br/pkg/storage" @@ -30,12 +32,30 @@ import ( type MDDatabaseMeta struct { Name string - SchemaFile string + SchemaFile FileInfo Tables []*MDTableMeta Views []*MDTableMeta charSet string } +func (m *MDDatabaseMeta) GetSchema(ctx context.Context, store storage.ExternalStorage) (string, error) { + schema, err := ExportStatement(ctx, store, m.SchemaFile, m.charSet) + if err != nil { + log.L().Warn("failed to extract table schema", + zap.String("Path", m.SchemaFile.FileMeta.Path), + log.ShortError(err), + ) + schema = nil + } + schemaStr := strings.TrimSpace(string(schema)) + // set default if schema sql is empty + if len(schemaStr) == 0 { + schemaStr = "CREATE DATABASE IF NOT EXISTS " + common.EscapeIdentifier(m.Name) + } + + return schemaStr, nil +} + type MDTableMeta struct { DB string Name string @@ -219,7 +239,7 @@ func (s *mdLoaderSetup) setup(ctx context.Context, store storage.ExternalStorage // setup database schema if len(s.dbSchemas) != 0 { for _, fileInfo := range s.dbSchemas { - if _, dbExists := s.insertDB(fileInfo.TableName.Schema, fileInfo.FileMeta.Path); dbExists && s.loader.router == nil { + if _, dbExists := s.insertDB(fileInfo); dbExists && s.loader.router == nil { return errors.Errorf("invalid database schema file, duplicated item - %s", fileInfo.FileMeta.Path) } } @@ -406,15 +426,15 @@ func (s *mdLoaderSetup) route() error { return nil } -func (s *mdLoaderSetup) insertDB(dbName string, path string) (*MDDatabaseMeta, bool) { - dbIndex, ok := s.dbIndexMap[dbName] +func (s *mdLoaderSetup) insertDB(f FileInfo) (*MDDatabaseMeta, bool) { + dbIndex, ok := s.dbIndexMap[f.TableName.Schema] if ok { return s.loader.dbs[dbIndex], true } - s.dbIndexMap[dbName] = len(s.loader.dbs) + s.dbIndexMap[f.TableName.Schema] = len(s.loader.dbs) ptr := &MDDatabaseMeta{ - Name: dbName, - SchemaFile: path, + Name: f.TableName.Schema, + SchemaFile: f, charSet: s.loader.charSet, } s.loader.dbs = append(s.loader.dbs, ptr) @@ -422,7 +442,13 @@ func (s *mdLoaderSetup) insertDB(dbName string, path string) (*MDDatabaseMeta, b } func (s *mdLoaderSetup) insertTable(fileInfo FileInfo) (*MDTableMeta, bool, bool) { - dbMeta, dbExists := s.insertDB(fileInfo.TableName.Schema, "") + dbFileInfo := FileInfo{ + TableName: filter.Table{ + Schema: fileInfo.TableName.Schema, + }, + FileMeta: SourceFileMeta{Type: SourceTypeSchemaSchema}, + } + dbMeta, dbExists := s.insertDB(dbFileInfo) tableIndex, ok := s.tableIndexMap[fileInfo.TableName] if ok { return dbMeta.Tables[tableIndex], dbExists, true @@ -442,7 +468,13 @@ func (s *mdLoaderSetup) insertTable(fileInfo FileInfo) (*MDTableMeta, bool, bool } func (s *mdLoaderSetup) insertView(fileInfo FileInfo) (bool, bool) { - dbMeta, dbExists := s.insertDB(fileInfo.TableName.Schema, "") + dbFileInfo := FileInfo{ + TableName: filter.Table{ + Schema: fileInfo.TableName.Schema, + }, + FileMeta: SourceFileMeta{Type: SourceTypeSchemaSchema}, + } + dbMeta, dbExists := s.insertDB(dbFileInfo) _, ok := s.tableIndexMap[fileInfo.TableName] if ok { meta := &MDTableMeta{ diff --git a/br/pkg/lightning/mydump/loader_test.go b/br/pkg/lightning/mydump/loader_test.go index 76bc50eba2793..08442cacffd86 100644 --- a/br/pkg/lightning/mydump/loader_test.go +++ b/br/pkg/lightning/mydump/loader_test.go @@ -179,6 +179,9 @@ func (s *testMydumpLoaderSuite) TestTableInfoNotFound(c *C) { loader, err := md.NewMyDumpLoader(ctx, s.cfg) c.Assert(err, IsNil) for _, dbMeta := range loader.GetDatabases() { + dbSQL, err := dbMeta.GetSchema(ctx, store) + c.Assert(err, IsNil) + c.Assert(dbSQL, Equals, "CREATE DATABASE IF NOT EXISTS `db`") for _, tblMeta := range dbMeta.Tables { sql, err := tblMeta.GetSchema(ctx, store) c.Assert(sql, Equals, "") @@ -272,8 +275,14 @@ func (s *testMydumpLoaderSuite) TestDataWithoutSchema(c *C) { mdl, err := md.NewMyDumpLoader(context.Background(), s.cfg) c.Assert(err, IsNil) c.Assert(mdl.GetDatabases(), DeepEquals, []*md.MDDatabaseMeta{{ - Name: "db", - SchemaFile: "", + Name: "db", + SchemaFile: md.FileInfo{ + TableName: filter.Table{ + Schema: "db", + Name: "", + }, + FileMeta: md.SourceFileMeta{Type: md.SourceTypeSchemaSchema}, + }, Tables: []*md.MDTableMeta{{ DB: "db", Name: "tbl", @@ -302,7 +311,7 @@ func (s *testMydumpLoaderSuite) TestTablesWithDots(c *C) { c.Assert(err, IsNil) c.Assert(mdl.GetDatabases(), DeepEquals, []*md.MDDatabaseMeta{{ Name: "db", - SchemaFile: "db-schema-create.sql", + SchemaFile: md.FileInfo{TableName: filter.Table{Schema: "db", Name: ""}, FileMeta: md.SourceFileMeta{Path: "db-schema-create.sql", Type: md.SourceTypeSchemaSchema}}, Tables: []*md.MDTableMeta{ { DB: "db", @@ -396,7 +405,7 @@ func (s *testMydumpLoaderSuite) TestRouter(c *C) { c.Assert(mdl.GetDatabases(), DeepEquals, []*md.MDDatabaseMeta{ { Name: "a1", - SchemaFile: "a1-schema-create.sql", + SchemaFile: md.FileInfo{TableName: filter.Table{Schema: "a1", Name: ""}, FileMeta: md.SourceFileMeta{Path: "a1-schema-create.sql", Type: md.SourceTypeSchemaSchema}}, Tables: []*md.MDTableMeta{ { DB: "a1", @@ -427,11 +436,11 @@ func (s *testMydumpLoaderSuite) TestRouter(c *C) { }, { Name: "d0", - SchemaFile: "d0-schema-create.sql", + SchemaFile: md.FileInfo{TableName: filter.Table{Schema: "d0", Name: ""}, FileMeta: md.SourceFileMeta{Path: "d0-schema-create.sql", Type: md.SourceTypeSchemaSchema}}, }, { Name: "b", - SchemaFile: "a0-schema-create.sql", + SchemaFile: md.FileInfo{TableName: filter.Table{Schema: "b", Name: ""}, FileMeta: md.SourceFileMeta{Path: "a0-schema-create.sql", Type: md.SourceTypeSchemaSchema}}, Tables: []*md.MDTableMeta{ { DB: "b", @@ -449,7 +458,7 @@ func (s *testMydumpLoaderSuite) TestRouter(c *C) { }, { Name: "c", - SchemaFile: "c0-schema-create.sql", + SchemaFile: md.FileInfo{TableName: filter.Table{Schema: "c", Name: ""}, FileMeta: md.SourceFileMeta{Path: "c0-schema-create.sql", Type: md.SourceTypeSchemaSchema}}, Tables: []*md.MDTableMeta{ { DB: "c", @@ -463,7 +472,7 @@ func (s *testMydumpLoaderSuite) TestRouter(c *C) { }, { Name: "v", - SchemaFile: "e0-schema-create.sql", + SchemaFile: md.FileInfo{TableName: filter.Table{Schema: "v", Name: ""}, FileMeta: md.SourceFileMeta{Path: "e0-schema-create.sql", Type: md.SourceTypeSchemaSchema}}, Tables: []*md.MDTableMeta{ { DB: "v", @@ -552,7 +561,7 @@ func (s *testMydumpLoaderSuite) TestFileRouting(c *C) { c.Assert(mdl.GetDatabases(), DeepEquals, []*md.MDDatabaseMeta{ { Name: "d1", - SchemaFile: filepath.FromSlash("d1/schema.sql"), + SchemaFile: md.FileInfo{TableName: filter.Table{Schema: "d1", Name: ""}, FileMeta: md.SourceFileMeta{Path: filepath.FromSlash("d1/schema.sql"), Type: md.SourceTypeSchemaSchema}}, Tables: []*md.MDTableMeta{ { DB: "d1", @@ -605,7 +614,7 @@ func (s *testMydumpLoaderSuite) TestFileRouting(c *C) { }, { Name: "d2", - SchemaFile: filepath.FromSlash("d2/schema.sql"), + SchemaFile: md.FileInfo{TableName: filter.Table{Schema: "d2", Name: ""}, FileMeta: md.SourceFileMeta{Path: filepath.FromSlash("d2/schema.sql"), Type: md.SourceTypeSchemaSchema}}, Tables: []*md.MDTableMeta{ { DB: "d2", diff --git a/br/pkg/lightning/restore/restore.go b/br/pkg/lightning/restore/restore.go index 75cb9b0b6729d..6a7eaf6b3550d 100644 --- a/br/pkg/lightning/restore/restore.go +++ b/br/pkg/lightning/restore/restore.go @@ -501,11 +501,7 @@ type schemaJob struct { dbName string tblName string // empty for create db jobs stmtType schemaStmtType - stmts []*schemaStmt -} - -type schemaStmt struct { - sql string + stmts []string } type restoreSchemaWorker struct { @@ -518,6 +514,15 @@ type restoreSchemaWorker struct { store storage.ExternalStorage } +func (worker *restoreSchemaWorker) addJob(sqlStr string, job *schemaJob) error { + stmts, err := createIfNotExistsStmt(worker.glue.GetParser(), sqlStr, job.dbName, job.tblName) + if err != nil { + return err + } + job.stmts = stmts + return worker.appendJob(job) +} + func (worker *restoreSchemaWorker) makeJobs( dbMetas []*mydump.MDDatabaseMeta, getTables func(context.Context, string) ([]*model.TableInfo, error), @@ -529,15 +534,15 @@ func (worker *restoreSchemaWorker) makeJobs( var err error // 1. restore databases, execute statements concurrency for _, dbMeta := range dbMetas { - restoreSchemaJob := &schemaJob{ + sql, err := dbMeta.GetSchema(worker.ctx, worker.store) + if err != nil { + return err + } + err = worker.addJob(sql, &schemaJob{ dbName: dbMeta.Name, + tblName: "", stmtType: schemaCreateDatabase, - stmts: make([]*schemaStmt, 0, 1), - } - restoreSchemaJob.stmts = append(restoreSchemaJob.stmts, &schemaStmt{ - sql: createDatabaseIfNotExistStmt(dbMeta.Name), }) - err = worker.appendJob(restoreSchemaJob) if err != nil { return err } @@ -563,30 +568,19 @@ func (worker *restoreSchemaWorker) makeJobs( return errors.Errorf("table `%s`.`%s` schema not found", dbMeta.Name, tblMeta.Name) } sql, err := tblMeta.GetSchema(worker.ctx, worker.store) + if err != nil { + return err + } if sql != "" { - stmts, err := createTableIfNotExistsStmt(worker.glue.GetParser(), sql, dbMeta.Name, tblMeta.Name) - if err != nil { - return err - } - restoreSchemaJob := &schemaJob{ + err = worker.addJob(sql, &schemaJob{ dbName: dbMeta.Name, tblName: tblMeta.Name, stmtType: schemaCreateTable, - stmts: make([]*schemaStmt, 0, len(stmts)), - } - for _, sql := range stmts { - restoreSchemaJob.stmts = append(restoreSchemaJob.stmts, &schemaStmt{ - sql: sql, - }) - } - err = worker.appendJob(restoreSchemaJob) + }) if err != nil { return err } } - if err != nil { - return err - } } } err = worker.wait() @@ -598,22 +592,11 @@ func (worker *restoreSchemaWorker) makeJobs( for _, viewMeta := range dbMeta.Views { sql, err := viewMeta.GetSchema(worker.ctx, worker.store) if sql != "" { - stmts, err := createTableIfNotExistsStmt(worker.glue.GetParser(), sql, dbMeta.Name, viewMeta.Name) - if err != nil { - return err - } - restoreSchemaJob := &schemaJob{ + err = worker.addJob(sql, &schemaJob{ dbName: dbMeta.Name, tblName: viewMeta.Name, stmtType: schemaCreateView, - stmts: make([]*schemaStmt, 0, len(stmts)), - } - for _, sql := range stmts { - restoreSchemaJob.stmts = append(restoreSchemaJob.stmts, &schemaStmt{ - sql: sql, - }) - } - err = worker.appendJob(restoreSchemaJob) + }) if err != nil { return err } @@ -674,8 +657,8 @@ loop: DB: session, } for _, stmt := range job.stmts { - task := logger.Begin(zap.DebugLevel, fmt.Sprintf("execute SQL: %s", stmt.sql)) - err = sqlWithRetry.Exec(worker.ctx, "run create schema job", stmt.sql) + task := logger.Begin(zap.DebugLevel, fmt.Sprintf("execute SQL: %s", stmt)) + err = sqlWithRetry.Exec(worker.ctx, "run create schema job", stmt) task.End(zap.ErrorLevel, err) if err != nil { err = errors.Annotatef(err, "%s %s failed", job.stmtType.String(), common.UniqueTable(job.dbName, job.tblName)) @@ -735,7 +718,7 @@ func (worker *restoreSchemaWorker) appendJob(job *schemaJob) error { case <-worker.ctx.Done(): // cancel the job worker.wg.Done() - return worker.ctx.Err() + return errors.Trace(worker.ctx.Err()) case worker.jobCh <- job: return nil } diff --git a/br/pkg/lightning/restore/tidb.go b/br/pkg/lightning/restore/tidb.go index 7ec0c2ae65cec..4616ae66ac71e 100644 --- a/br/pkg/lightning/restore/tidb.go +++ b/br/pkg/lightning/restore/tidb.go @@ -179,7 +179,7 @@ loopCreate: for tbl, sqlCreateTable := range tablesSchema { task.Debug("create table", zap.String("schema", sqlCreateTable)) - sqlCreateStmts, err = createTableIfNotExistsStmt(g.GetParser(), sqlCreateTable, database, tbl) + sqlCreateStmts, err = createIfNotExistsStmt(g.GetParser(), sqlCreateTable, database, tbl) if err != nil { break } @@ -202,14 +202,7 @@ loopCreate: return errors.Trace(err) } -func createDatabaseIfNotExistStmt(dbName string) string { - var createDatabase strings.Builder - createDatabase.WriteString("CREATE DATABASE IF NOT EXISTS ") - common.WriteMySQLIdentifier(&createDatabase, dbName) - return createDatabase.String() -} - -func createTableIfNotExistsStmt(p *parser.Parser, createTable, dbName, tblName string) ([]string, error) { +func createIfNotExistsStmt(p *parser.Parser, createTable, dbName, tblName string) ([]string, error) { stmts, _, err := p.ParseSQL(createTable) if err != nil { return []string{}, err @@ -221,6 +214,9 @@ func createTableIfNotExistsStmt(p *parser.Parser, createTable, dbName, tblName s retStmts := make([]string, 0, len(stmts)) for _, stmt := range stmts { switch node := stmt.(type) { + case *ast.CreateDatabaseStmt: + node.Name = dbName + node.IfNotExists = true case *ast.CreateTableStmt: node.Table.Schema = model.NewCIStr(dbName) node.Table.Name = model.NewCIStr(tblName) diff --git a/br/pkg/lightning/restore/tidb_test.go b/br/pkg/lightning/restore/tidb_test.go index 42b5a10b92e76..4599d64540d17 100644 --- a/br/pkg/lightning/restore/tidb_test.go +++ b/br/pkg/lightning/restore/tidb_test.go @@ -65,103 +65,109 @@ func (s *tidbSuite) TearDownTest(c *C) { func (s *tidbSuite) TestCreateTableIfNotExistsStmt(c *C) { dbName := "testdb" - createTableIfNotExistsStmt := func(createTable, tableName string) []string { - res, err := createTableIfNotExistsStmt(s.tiGlue.GetParser(), createTable, dbName, tableName) + createSQLIfNotExistsStmt := func(createTable, tableName string) []string { + res, err := createIfNotExistsStmt(s.tiGlue.GetParser(), createTable, dbName, tableName) c.Assert(err, IsNil) return res } c.Assert( - createTableIfNotExistsStmt("CREATE TABLE `foo`(`bar` TINYINT(1));", "foo"), + createSQLIfNotExistsStmt("CREATE DATABASE `foo` CHARACTER SET = utf8 COLLATE = utf8_general_ci;", ""), + DeepEquals, + []string{"CREATE DATABASE IF NOT EXISTS `testdb` CHARACTER SET = utf8 COLLATE = utf8_general_ci;"}, + ) + + c.Assert( + createSQLIfNotExistsStmt("CREATE TABLE `foo`(`bar` TINYINT(1));", "foo"), DeepEquals, []string{"CREATE TABLE IF NOT EXISTS `testdb`.`foo` (`bar` TINYINT(1));"}, ) c.Assert( - createTableIfNotExistsStmt("CREATE TABLE IF NOT EXISTS `foo`(`bar` TINYINT(1));", "foo"), + createSQLIfNotExistsStmt("CREATE TABLE IF NOT EXISTS `foo`(`bar` TINYINT(1));", "foo"), DeepEquals, []string{"CREATE TABLE IF NOT EXISTS `testdb`.`foo` (`bar` TINYINT(1));"}, ) // case insensitive c.Assert( - createTableIfNotExistsStmt("/* cOmmEnt */ creAte tablE `fOo`(`bar` TinyinT(1));", "fOo"), + createSQLIfNotExistsStmt("/* cOmmEnt */ creAte tablE `fOo`(`bar` TinyinT(1));", "fOo"), DeepEquals, []string{"CREATE TABLE IF NOT EXISTS `testdb`.`fOo` (`bar` TINYINT(1));"}, ) c.Assert( - createTableIfNotExistsStmt("/* coMMenT */ crEatE tAble If not EXISts `FoO`(`bAR` tiNyInT(1));", "FoO"), + createSQLIfNotExistsStmt("/* coMMenT */ crEatE tAble If not EXISts `FoO`(`bAR` tiNyInT(1));", "FoO"), DeepEquals, []string{"CREATE TABLE IF NOT EXISTS `testdb`.`FoO` (`bAR` TINYINT(1));"}, ) // only one "CREATE TABLE" is replaced c.Assert( - createTableIfNotExistsStmt("CREATE TABLE `foo`(`bar` INT(1) COMMENT 'CREATE TABLE');", "foo"), + createSQLIfNotExistsStmt("CREATE TABLE `foo`(`bar` INT(1) COMMENT 'CREATE TABLE');", "foo"), DeepEquals, []string{"CREATE TABLE IF NOT EXISTS `testdb`.`foo` (`bar` INT(1) COMMENT 'CREATE TABLE');"}, ) // test clustered index consistency c.Assert( - createTableIfNotExistsStmt("CREATE TABLE `foo`(`bar` INT(1) PRIMARY KEY CLUSTERED COMMENT 'CREATE TABLE');", "foo"), + createSQLIfNotExistsStmt("CREATE TABLE `foo`(`bar` INT(1) PRIMARY KEY CLUSTERED COMMENT 'CREATE TABLE');", "foo"), DeepEquals, []string{"CREATE TABLE IF NOT EXISTS `testdb`.`foo` (`bar` INT(1) PRIMARY KEY /*T![clustered_index] CLUSTERED */ COMMENT 'CREATE TABLE');"}, ) c.Assert( - createTableIfNotExistsStmt("CREATE TABLE `foo`(`bar` INT(1) COMMENT 'CREATE TABLE', PRIMARY KEY (`bar`) NONCLUSTERED);", "foo"), + createSQLIfNotExistsStmt("CREATE TABLE `foo`(`bar` INT(1) COMMENT 'CREATE TABLE', PRIMARY KEY (`bar`) NONCLUSTERED);", "foo"), DeepEquals, []string{"CREATE TABLE IF NOT EXISTS `testdb`.`foo` (`bar` INT(1) COMMENT 'CREATE TABLE',PRIMARY KEY(`bar`) /*T![clustered_index] NONCLUSTERED */);"}, ) c.Assert( - createTableIfNotExistsStmt("CREATE TABLE `foo`(`bar` INT(1) PRIMARY KEY /*T![clustered_index] NONCLUSTERED */ COMMENT 'CREATE TABLE');", "foo"), + createSQLIfNotExistsStmt("CREATE TABLE `foo`(`bar` INT(1) PRIMARY KEY /*T![clustered_index] NONCLUSTERED */ COMMENT 'CREATE TABLE');", "foo"), DeepEquals, []string{"CREATE TABLE IF NOT EXISTS `testdb`.`foo` (`bar` INT(1) PRIMARY KEY /*T![clustered_index] NONCLUSTERED */ COMMENT 'CREATE TABLE');"}, ) c.Assert( - createTableIfNotExistsStmt("CREATE TABLE `foo`(`bar` INT(1) COMMENT 'CREATE TABLE', PRIMARY KEY (`bar`) /*T![clustered_index] CLUSTERED */);", "foo"), + createSQLIfNotExistsStmt("CREATE TABLE `foo`(`bar` INT(1) COMMENT 'CREATE TABLE', PRIMARY KEY (`bar`) /*T![clustered_index] CLUSTERED */);", "foo"), DeepEquals, []string{"CREATE TABLE IF NOT EXISTS `testdb`.`foo` (`bar` INT(1) COMMENT 'CREATE TABLE',PRIMARY KEY(`bar`) /*T![clustered_index] CLUSTERED */);"}, ) c.Assert( - createTableIfNotExistsStmt("CREATE TABLE `foo`(`bar` INT(1) PRIMARY KEY AUTO_RANDOM(2) COMMENT 'CREATE TABLE');", "foo"), + createSQLIfNotExistsStmt("CREATE TABLE `foo`(`bar` INT(1) PRIMARY KEY AUTO_RANDOM(2) COMMENT 'CREATE TABLE');", "foo"), DeepEquals, []string{"CREATE TABLE IF NOT EXISTS `testdb`.`foo` (`bar` INT(1) PRIMARY KEY /*T![auto_rand] AUTO_RANDOM(2) */ COMMENT 'CREATE TABLE');"}, ) // upper case becomes shorter c.Assert( - createTableIfNotExistsStmt("CREATE TABLE `ſ`(`ı` TINYINT(1));", "ſ"), + createSQLIfNotExistsStmt("CREATE TABLE `ſ`(`ı` TINYINT(1));", "ſ"), DeepEquals, []string{"CREATE TABLE IF NOT EXISTS `testdb`.`ſ` (`ı` TINYINT(1));"}, ) // upper case becomes longer c.Assert( - createTableIfNotExistsStmt("CREATE TABLE `ɑ`(`ȿ` TINYINT(1));", "ɑ"), + createSQLIfNotExistsStmt("CREATE TABLE `ɑ`(`ȿ` TINYINT(1));", "ɑ"), DeepEquals, []string{"CREATE TABLE IF NOT EXISTS `testdb`.`ɑ` (`ȿ` TINYINT(1));"}, ) // non-utf-8 c.Assert( - createTableIfNotExistsStmt("CREATE TABLE `\xcc\xcc\xcc`(`\xdd\xdd\xdd` TINYINT(1));", "\xcc\xcc\xcc"), + createSQLIfNotExistsStmt("CREATE TABLE `\xcc\xcc\xcc`(`\xdd\xdd\xdd` TINYINT(1));", "\xcc\xcc\xcc"), DeepEquals, []string{"CREATE TABLE IF NOT EXISTS `testdb`.`\xcc\xcc\xcc` (`???` TINYINT(1));"}, ) // renaming a table c.Assert( - createTableIfNotExistsStmt("create table foo(x int);", "ba`r"), + createSQLIfNotExistsStmt("create table foo(x int);", "ba`r"), DeepEquals, []string{"CREATE TABLE IF NOT EXISTS `testdb`.`ba``r` (`x` INT);"}, ) // conditional comments c.Assert( - createTableIfNotExistsStmt(` + createSQLIfNotExistsStmt(` /*!40101 SET NAMES binary*/; /*!40014 SET FOREIGN_KEY_CHECKS=0*/; CREATE TABLE x.y (z double) ENGINE=InnoDB AUTO_INCREMENT=8343230 DEFAULT CHARSET=utf8; @@ -176,7 +182,7 @@ func (s *tidbSuite) TestCreateTableIfNotExistsStmt(c *C) { // create view c.Assert( - createTableIfNotExistsStmt(` + createSQLIfNotExistsStmt(` /*!40101 SET NAMES binary*/; DROP TABLE IF EXISTS v2; DROP VIEW IF EXISTS v2; diff --git a/br/tests/lightning_column_permutation/data/perm-schema-create.sql b/br/tests/lightning_column_permutation/data/perm-schema-create.sql index fe9a5be60a3ff..28138f8d72659 100644 --- a/br/tests/lightning_column_permutation/data/perm-schema-create.sql +++ b/br/tests/lightning_column_permutation/data/perm-schema-create.sql @@ -1 +1 @@ -CREATE DATABASE `perm` IF NOT EXISTS; +CREATE DATABASE IF NOT EXISTS `perm`; diff --git a/br/tests/lightning_new_collation/data/nc-schema-create.sql b/br/tests/lightning_new_collation/data/nc-schema-create.sql new file mode 100644 index 0000000000000..6608189c71304 --- /dev/null +++ b/br/tests/lightning_new_collation/data/nc-schema-create.sql @@ -0,0 +1 @@ +CREATE DATABASE nc CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci; diff --git a/br/tests/lightning_new_collation/data/nc.ci-schema.sql b/br/tests/lightning_new_collation/data/nc.ci-schema.sql new file mode 100644 index 0000000000000..1e7958a76409c --- /dev/null +++ b/br/tests/lightning_new_collation/data/nc.ci-schema.sql @@ -0,0 +1 @@ +CREATE TABLE ci(i INT PRIMARY KEY, v varchar(32)); diff --git a/br/tests/lightning_new_collation/data/nc.ci.0.csv b/br/tests/lightning_new_collation/data/nc.ci.0.csv new file mode 100644 index 0000000000000..a1b4dcff21e40 --- /dev/null +++ b/br/tests/lightning_new_collation/data/nc.ci.0.csv @@ -0,0 +1,2 @@ +i,v +1,aA diff --git a/br/tests/lightning_new_collation/run.sh b/br/tests/lightning_new_collation/run.sh index d4c49e3c61192..f360ed3a94fc4 100644 --- a/br/tests/lightning_new_collation/run.sh +++ b/br/tests/lightning_new_collation/run.sh @@ -54,6 +54,10 @@ for BACKEND in local importer tidb; do run_sql "SELECT j FROM nc.t WHERE s = 'This_Is_Test4'"; check_contains "j: 4" + run_sql "SELeCT i, v from nc.ci where v = 'aa';" + check_contains "i: 1" + check_contains "v: aA" + done # restart with original config