From 5a08acdf19c0400e6a07b1c142212ff4024c4fb3 Mon Sep 17 00:00:00 2001 From: mychoxin Date: Tue, 20 Dec 2022 10:34:57 +0800 Subject: [PATCH 1/5] docs: fix 'enable_global_index' to 'enable-global-index' (#40009) --- docs/design/2020-08-04-global-index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/design/2020-08-04-global-index.md b/docs/design/2020-08-04-global-index.md index 80078688777b7..f5e2d89f932c4 100644 --- a/docs/design/2020-08-04-global-index.md +++ b/docs/design/2020-08-04-global-index.md @@ -183,7 +183,7 @@ In TiDB, operators in the partitioned table will be translated to UnionAll in th ## Compatibility -MySQL does not support global index, which means this feature may cause some compatibility issues. We add an option `enable_global_index` in `config.Config` to control it. The default value of this option is `false`, so TiDB will keep consistent with MySQL, unless the user open global index feature manually. +MySQL does not support global index, which means this feature may cause some compatibility issues. We add an option `enable-global-index` in `config.Config` to control it. The default value of this option is `false`, so TiDB will keep consistent with MySQL, unless the user open global index feature manually. ## Implementation From ae58fa1c8bca029907e526cc8c0812e60bbb30f7 Mon Sep 17 00:00:00 2001 From: xiongjiwei Date: Tue, 20 Dec 2022 12:56:54 +0800 Subject: [PATCH 2/5] expression: introduce `cast(... as ... array)` in expression index (#39992) close pingcap/tidb#40045 --- br/pkg/lightning/backend/kv/sql2kv.go | 2 +- ddl/ddl_api.go | 6 +- ddl/generated_column.go | 24 ++++++-- ddl/index.go | 21 ++++--- ddl/partition.go | 2 +- expression/BUILD.bazel | 1 + expression/builtin_cast.go | 83 +++++++++++++++++++++++++-- expression/errors.go | 1 + expression/expression.go | 4 +- expression/multi_valued_index_test.go | 47 +++++++++++++++ expression/simple_rewriter.go | 8 +-- parser/model/model.go | 1 + parser/parser_test.go | 1 + parser/types/field_type.go | 53 ++++++++++------- planner/core/expression_rewriter.go | 15 ++++- planner/core/planbuilder.go | 11 ++++ types/field_type_builder.go | 6 ++ 17 files changed, 232 insertions(+), 54 deletions(-) create mode 100644 expression/multi_valued_index_test.go diff --git a/br/pkg/lightning/backend/kv/sql2kv.go b/br/pkg/lightning/backend/kv/sql2kv.go index 6cebb1e29e329..9ad552ef5f340 100644 --- a/br/pkg/lightning/backend/kv/sql2kv.go +++ b/br/pkg/lightning/backend/kv/sql2kv.go @@ -169,7 +169,7 @@ func collectGeneratedColumns(se *session, meta *model.TableInfo, cols []*table.C var genCols []genCol for i, col := range cols { if col.GeneratedExpr != nil { - expr, err := expression.RewriteAstExpr(se, col.GeneratedExpr, schema, names) + expr, err := expression.RewriteAstExpr(se, col.GeneratedExpr, schema, names, false) if err != nil { return nil, err } diff --git a/ddl/ddl_api.go b/ddl/ddl_api.go index 2850a3aa968a5..429d3f13425b2 100644 --- a/ddl/ddl_api.go +++ b/ddl/ddl_api.go @@ -6164,7 +6164,7 @@ func (d *ddl) CreatePrimaryKey(ctx sessionctx.Context, ti ast.Ident, indexName m // After DDL job is put to the queue, and if the check fail, TiDB will run the DDL cancel logic. // The recover step causes DDL wait a few seconds, makes the unit test painfully slow. // For same reason, decide whether index is global here. - indexColumns, err := buildIndexColumns(ctx, tblInfo.Columns, indexPartSpecifications) + indexColumns, _, err := buildIndexColumns(ctx, tblInfo.Columns, indexPartSpecifications) if err != nil { return errors.Trace(err) } @@ -6274,7 +6274,7 @@ func BuildHiddenColumnInfo(ctx sessionctx.Context, indexPartSpecifications []*as if err != nil { return nil, errors.Trace(err) } - expr, err := expression.RewriteSimpleExprWithTableInfo(ctx, tblInfo, idxPart.Expr) + expr, err := expression.RewriteSimpleExprWithTableInfo(ctx, tblInfo, idxPart.Expr, true) if err != nil { // TODO: refine the error message. return nil, err @@ -6389,7 +6389,7 @@ func (d *ddl) createIndex(ctx sessionctx.Context, ti ast.Ident, keyType ast.Inde // After DDL job is put to the queue, and if the check fail, TiDB will run the DDL cancel logic. // The recover step causes DDL wait a few seconds, makes the unit test painfully slow. // For same reason, decide whether index is global here. - indexColumns, err := buildIndexColumns(ctx, finalColumns, indexPartSpecifications) + indexColumns, _, err := buildIndexColumns(ctx, finalColumns, indexPartSpecifications) if err != nil { return errors.Trace(err) } diff --git a/ddl/generated_column.go b/ddl/generated_column.go index 2f4ceee8b60a9..678d803edf521 100644 --- a/ddl/generated_column.go +++ b/ddl/generated_column.go @@ -268,12 +268,14 @@ func checkModifyGeneratedColumn(sctx sessionctx.Context, tbl table.Table, oldCol } type illegalFunctionChecker struct { - hasIllegalFunc bool - hasAggFunc bool - hasRowVal bool // hasRowVal checks whether the functional index refers to a row value - hasWindowFunc bool - hasNotGAFunc4ExprIdx bool - otherErr error + hasIllegalFunc bool + hasAggFunc bool + hasRowVal bool // hasRowVal checks whether the functional index refers to a row value + hasWindowFunc bool + hasNotGAFunc4ExprIdx bool + hasCastArrayFunc bool + disallowCastArrayFunc bool + otherErr error } func (c *illegalFunctionChecker) Enter(inNode ast.Node) (outNode ast.Node, skipChildren bool) { @@ -308,7 +310,14 @@ func (c *illegalFunctionChecker) Enter(inNode ast.Node) (outNode ast.Node, skipC case *ast.WindowFuncExpr: c.hasWindowFunc = true return inNode, true + case *ast.FuncCastExpr: + c.hasCastArrayFunc = c.hasCastArrayFunc || node.Tp.IsArray() + if c.disallowCastArrayFunc && node.Tp.IsArray() { + c.otherErr = expression.ErrNotSupportedYet.GenWithStackByArgs("Use of CAST( .. AS .. ARRAY) outside of functional index in CREATE(non-SELECT)/ALTER TABLE or in general expressions") + return inNode, true + } } + c.disallowCastArrayFunc = true return inNode, false } @@ -355,6 +364,9 @@ func checkIllegalFn4Generated(name string, genType int, expr ast.ExprNode) error if genType == typeIndex && c.hasNotGAFunc4ExprIdx && !config.GetGlobalConfig().Experimental.AllowsExpressionIndex { return dbterror.ErrUnsupportedExpressionIndex } + if genType == typeColumn && c.hasCastArrayFunc { + return expression.ErrNotSupportedYet.GenWithStackByArgs("Use of CAST( .. AS .. ARRAY) outside of functional index in CREATE(non-SELECT)/ALTER TABLE or in general expressions") + } return nil } diff --git a/ddl/index.go b/ddl/index.go index 0f70b73b61046..273b89e041233 100644 --- a/ddl/index.go +++ b/ddl/index.go @@ -64,26 +64,28 @@ var ( telemetryAddIndexIngestUsage = metrics.TelemetryAddIndexIngestCnt ) -func buildIndexColumns(ctx sessionctx.Context, columns []*model.ColumnInfo, indexPartSpecifications []*ast.IndexPartSpecification) ([]*model.IndexColumn, error) { +func buildIndexColumns(ctx sessionctx.Context, columns []*model.ColumnInfo, indexPartSpecifications []*ast.IndexPartSpecification) ([]*model.IndexColumn, bool, error) { // Build offsets. idxParts := make([]*model.IndexColumn, 0, len(indexPartSpecifications)) var col *model.ColumnInfo + var mvIndex bool maxIndexLength := config.GetGlobalConfig().MaxIndexLength // The sum of length of all index columns. sumLength := 0 for _, ip := range indexPartSpecifications { col = model.FindColumnInfo(columns, ip.Column.Name.L) if col == nil { - return nil, dbterror.ErrKeyColumnDoesNotExits.GenWithStack("column does not exist: %s", ip.Column.Name) + return nil, false, dbterror.ErrKeyColumnDoesNotExits.GenWithStack("column does not exist: %s", ip.Column.Name) } if err := checkIndexColumn(ctx, col, ip.Length); err != nil { - return nil, err + return nil, false, err } + mvIndex = mvIndex || col.FieldType.IsArray() indexColLen := ip.Length indexColumnLength, err := getIndexColumnLength(col, ip.Length) if err != nil { - return nil, err + return nil, false, err } sumLength += indexColumnLength @@ -92,12 +94,12 @@ func buildIndexColumns(ctx sessionctx.Context, columns []*model.ColumnInfo, inde // The multiple column index and the unique index in which the length sum exceeds the maximum size // will return an error instead produce a warning. if ctx == nil || ctx.GetSessionVars().StrictSQLMode || mysql.HasUniKeyFlag(col.GetFlag()) || len(indexPartSpecifications) > 1 { - return nil, dbterror.ErrTooLongKey.GenWithStackByArgs(maxIndexLength) + return nil, false, dbterror.ErrTooLongKey.GenWithStackByArgs(maxIndexLength) } // truncate index length and produce warning message in non-restrict sql mode. colLenPerUint, err := getIndexColumnLength(col, 1) if err != nil { - return nil, err + return nil, false, err } indexColLen = maxIndexLength / colLenPerUint // produce warning message @@ -111,7 +113,7 @@ func buildIndexColumns(ctx sessionctx.Context, columns []*model.ColumnInfo, inde }) } - return idxParts, nil + return idxParts, mvIndex, nil } // CheckPKOnGeneratedColumn checks the specification of PK is valid. @@ -154,7 +156,7 @@ func checkIndexColumn(ctx sessionctx.Context, col *model.ColumnInfo, indexColumn } // JSON column cannot index. - if col.FieldType.GetType() == mysql.TypeJSON { + if col.FieldType.GetType() == mysql.TypeJSON && !col.FieldType.IsArray() { if col.Hidden { return dbterror.ErrFunctionalIndexOnJSONOrGeometryFunction } @@ -263,7 +265,7 @@ func BuildIndexInfo( return nil, errors.Trace(err) } - idxColumns, err := buildIndexColumns(ctx, allTableColumns, indexPartSpecifications) + idxColumns, mvIndex, err := buildIndexColumns(ctx, allTableColumns, indexPartSpecifications) if err != nil { return nil, errors.Trace(err) } @@ -276,6 +278,7 @@ func BuildIndexInfo( Primary: isPrimary, Unique: isUnique, Global: isGlobal, + MVIndex: mvIndex, } if indexOption != nil { diff --git a/ddl/partition.go b/ddl/partition.go index 0a1ea4e6fbe66..2c95f389707f9 100644 --- a/ddl/partition.go +++ b/ddl/partition.go @@ -1375,7 +1375,7 @@ func checkPartitionFuncType(ctx sessionctx.Context, expr ast.ExprNode, tblInfo * return nil } - e, err := expression.RewriteSimpleExprWithTableInfo(ctx, tblInfo, expr) + e, err := expression.RewriteSimpleExprWithTableInfo(ctx, tblInfo, expr, false) if err != nil { return errors.Trace(err) } diff --git a/expression/BUILD.bazel b/expression/BUILD.bazel index c7304642c544a..5a201d906b5a3 100644 --- a/expression/BUILD.bazel +++ b/expression/BUILD.bazel @@ -177,6 +177,7 @@ go_test( "integration_serial_test.go", "integration_test.go", "main_test.go", + "multi_valued_index_test.go", "scalar_function_test.go", "schema_test.go", "typeinfer_test.go", diff --git a/expression/builtin_cast.go b/expression/builtin_cast.go index ee66669e638d6..e6257c4dd058c 100644 --- a/expression/builtin_cast.go +++ b/expression/builtin_cast.go @@ -23,6 +23,7 @@ package expression import ( + "fmt" "math" "strconv" "strings" @@ -407,6 +408,70 @@ func (c *castAsDurationFunctionClass) getFunction(ctx sessionctx.Context, args [ return sig, nil } +type castAsArrayFunctionClass struct { + baseFunctionClass + + tp *types.FieldType +} + +func (c *castAsArrayFunctionClass) verifyArgs(args []Expression) error { + if err := c.baseFunctionClass.verifyArgs(args); err != nil { + return err + } + + if args[0].GetType().EvalType() != types.ETJson { + return types.ErrInvalidJSONData.GenWithStackByArgs("1", "cast_as_array") + } + + return nil +} + +func (c *castAsArrayFunctionClass) getFunction(ctx sessionctx.Context, args []Expression) (sig builtinFunc, err error) { + if err := c.verifyArgs(args); err != nil { + return nil, err + } + arrayType := c.tp.ArrayType() + switch arrayType.GetType() { + case mysql.TypeYear, mysql.TypeJSON: + return nil, ErrNotSupportedYet.GenWithStackByArgs(fmt.Sprintf("CAST-ing data to array of %s", arrayType.String())) + } + if arrayType.EvalType() == types.ETString && arrayType.GetCharset() != charset.CharsetUTF8MB4 && arrayType.GetCharset() != charset.CharsetBin { + return nil, ErrNotSupportedYet.GenWithStackByArgs("specifying charset for multi-valued index", arrayType.String()) + } + + bf, err := newBaseBuiltinFunc(ctx, c.funcName, args, c.tp) + if err != nil { + return nil, err + } + sig = &castJSONAsArrayFunctionSig{bf} + return sig, nil +} + +type castJSONAsArrayFunctionSig struct { + baseBuiltinFunc +} + +func (b *castJSONAsArrayFunctionSig) Clone() builtinFunc { + newSig := &castJSONAsArrayFunctionSig{} + newSig.cloneFrom(&b.baseBuiltinFunc) + return newSig +} + +func (b *castJSONAsArrayFunctionSig) evalJSON(row chunk.Row) (res types.BinaryJSON, isNull bool, err error) { + val, isNull, err := b.args[0].EvalJSON(b.ctx, row) + if isNull || err != nil { + return res, isNull, err + } + + if val.TypeCode != types.JSONTypeCodeArray { + return types.BinaryJSON{}, false, ErrNotSupportedYet.GenWithStackByArgs("CAST-ing Non-JSON Array type to array") + } + + // TODO: impl the cast(... as ... array) function + + return types.BinaryJSON{}, false, nil +} + type castAsJSONFunctionClass struct { baseFunctionClass @@ -1914,6 +1979,13 @@ func BuildCastCollationFunction(ctx sessionctx.Context, expr Expression, ec *Exp // BuildCastFunction builds a CAST ScalarFunction from the Expression. func BuildCastFunction(ctx sessionctx.Context, expr Expression, tp *types.FieldType) (res Expression) { + res, err := BuildCastFunctionWithCheck(ctx, expr, tp) + terror.Log(err) + return +} + +// BuildCastFunctionWithCheck builds a CAST ScalarFunction from the Expression and return error if any. +func BuildCastFunctionWithCheck(ctx sessionctx.Context, expr Expression, tp *types.FieldType) (res Expression, err error) { argType := expr.GetType() // If source argument's nullable, then target type should be nullable if !mysql.HasNotNullFlag(argType.GetFlag()) { @@ -1933,7 +2005,11 @@ func BuildCastFunction(ctx sessionctx.Context, expr Expression, tp *types.FieldT case types.ETDuration: fc = &castAsDurationFunctionClass{baseFunctionClass{ast.Cast, 1, 1}, tp} case types.ETJson: - fc = &castAsJSONFunctionClass{baseFunctionClass{ast.Cast, 1, 1}, tp} + if tp.IsArray() { + fc = &castAsArrayFunctionClass{baseFunctionClass{ast.Cast, 1, 1}, tp} + } else { + fc = &castAsJSONFunctionClass{baseFunctionClass{ast.Cast, 1, 1}, tp} + } case types.ETString: fc = &castAsStringFunctionClass{baseFunctionClass{ast.Cast, 1, 1}, tp} if expr.GetType().GetType() == mysql.TypeBit { @@ -1941,7 +2017,6 @@ func BuildCastFunction(ctx sessionctx.Context, expr Expression, tp *types.FieldT } } f, err := fc.getFunction(ctx, []Expression{expr}) - terror.Log(err) res = &ScalarFunction{ FuncName: model.NewCIStr(ast.Cast), RetType: tp, @@ -1950,10 +2025,10 @@ func BuildCastFunction(ctx sessionctx.Context, expr Expression, tp *types.FieldT // We do not fold CAST if the eval type of this scalar function is ETJson // since we may reset the flag of the field type of CastAsJson later which // would affect the evaluation of it. - if tp.EvalType() != types.ETJson { + if tp.EvalType() != types.ETJson && err == nil { res = FoldConstant(res) } - return res + return res, err } // WrapWithCastAsInt wraps `expr` with `cast` if the return type of expr is not diff --git a/expression/errors.go b/expression/errors.go index 0db38645f78d4..c56737ec2fae3 100644 --- a/expression/errors.go +++ b/expression/errors.go @@ -37,6 +37,7 @@ var ( ErrInvalidTableSample = dbterror.ClassExpression.NewStd(mysql.ErrInvalidTableSample) ErrInternal = dbterror.ClassOptimizer.NewStd(mysql.ErrInternal) ErrNoDB = dbterror.ClassOptimizer.NewStd(mysql.ErrNoDB) + ErrNotSupportedYet = dbterror.ClassExpression.NewStd(mysql.ErrNotSupportedYet) // All the un-exported errors are defined here: errFunctionNotExists = dbterror.ClassExpression.NewStd(mysql.ErrSpDoesNotExist) diff --git a/expression/expression.go b/expression/expression.go index 024bac00ef960..352f105c52d65 100644 --- a/expression/expression.go +++ b/expression/expression.go @@ -59,7 +59,7 @@ var EvalAstExpr func(sctx sessionctx.Context, expr ast.ExprNode) (types.Datum, e // RewriteAstExpr rewrites ast expression directly. // Note: initialized in planner/core // import expression and planner/core together to use EvalAstExpr -var RewriteAstExpr func(sctx sessionctx.Context, expr ast.ExprNode, schema *Schema, names types.NameSlice) (Expression, error) +var RewriteAstExpr func(sctx sessionctx.Context, expr ast.ExprNode, schema *Schema, names types.NameSlice, allowCastArray bool) (Expression, error) // VecExpr contains all vectorized evaluation methods. type VecExpr interface { @@ -998,7 +998,7 @@ func ColumnInfos2ColumnsAndNames(ctx sessionctx.Context, dbName, tblName model.C if err != nil { return nil, nil, errors.Trace(err) } - e, err := RewriteAstExpr(ctx, expr, mockSchema, names) + e, err := RewriteAstExpr(ctx, expr, mockSchema, names, false) if err != nil { return nil, nil, errors.Trace(err) } diff --git a/expression/multi_valued_index_test.go b/expression/multi_valued_index_test.go new file mode 100644 index 0000000000000..058d955faa4fb --- /dev/null +++ b/expression/multi_valued_index_test.go @@ -0,0 +1,47 @@ +// Copyright 2022 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package expression_test + +import ( + "testing" + + "github.com/pingcap/tidb/errno" + "github.com/pingcap/tidb/testkit" +) + +func TestMultiValuedIndexDDL(t *testing.T) { + store := testkit.CreateMockStore(t) + + tk := testkit.NewTestKit(t, store) + tk.MustExec("USE test;") + + tk.MustExec("create table t(a json);") + tk.MustGetErrCode("select cast(a as signed array) from t", errno.ErrNotSupportedYet) + tk.MustGetErrCode("select json_extract(cast(a as signed array), '$[0]') from t", errno.ErrNotSupportedYet) + tk.MustGetErrCode("select * from t where cast(a as signed array)", errno.ErrNotSupportedYet) + tk.MustGetErrCode("select cast('[1,2,3]' as unsigned array);", errno.ErrNotSupportedYet) + + tk.MustExec("drop table t") + tk.MustGetErrCode("CREATE TABLE t(x INT, KEY k ((1 AND CAST(JSON_ARRAY(x) AS UNSIGNED ARRAY))));", errno.ErrNotSupportedYet) + tk.MustGetErrCode("CREATE TABLE t1 (f1 json, key mvi((cast(cast(f1 as unsigned array) as unsigned array))));", errno.ErrNotSupportedYet) + tk.MustGetErrCode("CREATE TABLE t1 (f1 json, key mvi((cast(f1->>'$[*]' as unsigned array))));", errno.ErrInvalidJSONData) + tk.MustGetErrCode("CREATE TABLE t1 (f1 json, key mvi((cast(f1->'$[*]' as year array))));", errno.ErrNotSupportedYet) + tk.MustGetErrCode("CREATE TABLE t1 (f1 json, key mvi((cast(f1->'$[*]' as json array))));", errno.ErrNotSupportedYet) + tk.MustGetErrCode("CREATE TABLE t1 (f1 json, key mvi((cast(f1->'$[*]' as char(10) charset gbk array))));", errno.ErrNotSupportedYet) + tk.MustGetErrCode("create table t(j json, gc json as ((concat(cast(j->'$[*]' as unsigned array),\"x\"))));", errno.ErrNotSupportedYet) + tk.MustGetErrCode("create table t(j json, gc json as (cast(j->'$[*]' as unsigned array)));", errno.ErrNotSupportedYet) + tk.MustGetErrCode("create view v as select cast('[1,2,3]' as unsigned array);", errno.ErrNotSupportedYet) + tk.MustExec("create table t(a json, index idx((cast(a as signed array))));") +} diff --git a/expression/simple_rewriter.go b/expression/simple_rewriter.go index 808db9f69b4cf..3343a0cbaa169 100644 --- a/expression/simple_rewriter.go +++ b/expression/simple_rewriter.go @@ -48,7 +48,7 @@ func ParseSimpleExprWithTableInfo(ctx sessionctx.Context, exprStr string, tableI return nil, errors.Trace(err) } expr := stmts[0].(*ast.SelectStmt).Fields.Fields[0].Expr - return RewriteSimpleExprWithTableInfo(ctx, tableInfo, expr) + return RewriteSimpleExprWithTableInfo(ctx, tableInfo, expr, false) } // ParseSimpleExprCastWithTableInfo parses simple expression string to Expression. @@ -63,13 +63,13 @@ func ParseSimpleExprCastWithTableInfo(ctx sessionctx.Context, exprStr string, ta } // RewriteSimpleExprWithTableInfo rewrites simple ast.ExprNode to expression.Expression. -func RewriteSimpleExprWithTableInfo(ctx sessionctx.Context, tbl *model.TableInfo, expr ast.ExprNode) (Expression, error) { +func RewriteSimpleExprWithTableInfo(ctx sessionctx.Context, tbl *model.TableInfo, expr ast.ExprNode, allowCastArray bool) (Expression, error) { dbName := model.NewCIStr(ctx.GetSessionVars().CurrentDB) columns, names, err := ColumnInfos2ColumnsAndNames(ctx, dbName, tbl.Name, tbl.Cols(), tbl) if err != nil { return nil, err } - e, err := RewriteAstExpr(ctx, expr, NewSchema(columns...), names) + e, err := RewriteAstExpr(ctx, expr, NewSchema(columns...), names, allowCastArray) if err != nil { return nil, err } @@ -111,7 +111,7 @@ func ParseSimpleExprsWithNames(ctx sessionctx.Context, exprStr string, schema *S // RewriteSimpleExprWithNames rewrites simple ast.ExprNode to expression.Expression. func RewriteSimpleExprWithNames(ctx sessionctx.Context, expr ast.ExprNode, schema *Schema, names []*types.FieldName) (Expression, error) { - e, err := RewriteAstExpr(ctx, expr, schema, names) + e, err := RewriteAstExpr(ctx, expr, schema, names, false) if err != nil { return nil, err } diff --git a/parser/model/model.go b/parser/model/model.go index ba7c46bcd6333..19aabf4a06572 100644 --- a/parser/model/model.go +++ b/parser/model/model.go @@ -1419,6 +1419,7 @@ type IndexInfo struct { Primary bool `json:"is_primary"` // Whether the index is primary key. Invisible bool `json:"is_invisible"` // Whether the index is invisible. Global bool `json:"is_global"` // Whether the index is global. + MVIndex bool `json:"mv_index"` // Whether the index is multivalued index. } // Clone clones IndexInfo. diff --git a/parser/parser_test.go b/parser/parser_test.go index c06d2076f085a..7b72117f69d16 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -1603,6 +1603,7 @@ func TestBuiltin(t *testing.T) { {"select cast(time '2000' as year);", true, "SELECT CAST(TIME '2000' AS YEAR)"}, {"select cast(b as signed array);", true, "SELECT CAST(`b` AS SIGNED ARRAY)"}, + {"select cast(b as char(10) array);", true, "SELECT CAST(`b` AS CHAR(10) ARRAY)"}, // for last_insert_id {"SELECT last_insert_id();", true, "SELECT LAST_INSERT_ID()"}, diff --git a/parser/types/field_type.go b/parser/types/field_type.go index 369ed59fa7a59..ff0ac9793cf17 100644 --- a/parser/types/field_type.go +++ b/parser/types/field_type.go @@ -72,7 +72,7 @@ func NewFieldType(tp byte) *FieldType { // IsDecimalValid checks whether the decimal is valid. func (ft *FieldType) IsDecimalValid() bool { - if ft.tp == mysql.TypeNewDecimal && (ft.decimal < 0 || ft.decimal > mysql.MaxDecimalScale || ft.flen <= 0 || ft.flen > mysql.MaxDecimalWidth || ft.flen < ft.decimal) { + if ft.GetType() == mysql.TypeNewDecimal && (ft.decimal < 0 || ft.decimal > mysql.MaxDecimalScale || ft.flen <= 0 || ft.flen > mysql.MaxDecimalWidth || ft.flen < ft.decimal) { return false } return true @@ -80,7 +80,7 @@ func (ft *FieldType) IsDecimalValid() bool { // IsVarLengthType Determine whether the column type is a variable-length type func (ft *FieldType) IsVarLengthType() bool { - switch ft.tp { + switch ft.GetType() { case mysql.TypeVarchar, mysql.TypeVarString, mysql.TypeJSON, mysql.TypeBlob, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob: return true default: @@ -90,6 +90,9 @@ func (ft *FieldType) IsVarLengthType() bool { // GetType returns the type of the FieldType. func (ft *FieldType) GetType() byte { + if ft.array { + return mysql.TypeJSON + } return ft.tp } @@ -126,6 +129,7 @@ func (ft *FieldType) GetElems() []string { // SetType sets the type of the FieldType. func (ft *FieldType) SetType(tp byte) { ft.tp = tp + ft.array = false } // SetFlag sets the flag of the FieldType. @@ -160,7 +164,7 @@ func (ft *FieldType) SetFlen(flen int) { // SetFlenUnderLimit sets the length of the field to the value of the argument func (ft *FieldType) SetFlenUnderLimit(flen int) { - if ft.tp == mysql.TypeNewDecimal { + if ft.GetType() == mysql.TypeNewDecimal { ft.flen = mathutil.Min(flen, mysql.MaxDecimalWidth) } else { ft.flen = flen @@ -174,7 +178,7 @@ func (ft *FieldType) SetDecimal(decimal int) { // SetDecimalUnderLimit sets the decimal of the field to the value of the argument func (ft *FieldType) SetDecimalUnderLimit(decimal int) { - if ft.tp == mysql.TypeNewDecimal { + if ft.GetType() == mysql.TypeNewDecimal { ft.decimal = mathutil.Min(decimal, mysql.MaxDecimalScale) } else { ft.decimal = decimal @@ -183,7 +187,7 @@ func (ft *FieldType) SetDecimalUnderLimit(decimal int) { // UpdateFlenAndDecimalUnderLimit updates the length and decimal to the value of the argument func (ft *FieldType) UpdateFlenAndDecimalUnderLimit(old *FieldType, deltaDecimal int, deltaFlen int) { - if ft.tp != mysql.TypeNewDecimal { + if ft.GetType() != mysql.TypeNewDecimal { return } if old.decimal < 0 { @@ -229,6 +233,13 @@ func (ft *FieldType) IsArray() bool { return ft.array } +// ArrayType return the type of the array. +func (ft *FieldType) ArrayType() *FieldType { + clone := ft.Clone() + clone.SetArray(false) + return clone +} + // SetElemWithIsBinaryLit sets the element of the FieldType. func (ft *FieldType) SetElemWithIsBinaryLit(idx int, element string, isBinaryLit bool) { ft.elems[idx] = element @@ -274,7 +285,7 @@ func (ft *FieldType) Equal(other *FieldType) bool { // When tp is float or double with decimal unspecified, do not check whether flen is equal, // because flen for them is useless. // The decimal field can be ignored if the type is int or string. - tpEqual := (ft.tp == other.tp) || (ft.tp == mysql.TypeVarchar && other.tp == mysql.TypeVarString) || (ft.tp == mysql.TypeVarString && other.tp == mysql.TypeVarchar) + tpEqual := (ft.GetType() == other.GetType()) || (ft.GetType() == mysql.TypeVarchar && other.GetType() == mysql.TypeVarString) || (ft.GetType() == mysql.TypeVarString && other.GetType() == mysql.TypeVarchar) flenEqual := ft.flen == other.flen || (ft.EvalType() == ETReal && ft.decimal == UnspecifiedLength) ignoreDecimal := ft.EvalType() == ETInt || ft.EvalType() == ETString partialEqual := tpEqual && @@ -316,7 +327,7 @@ func (ft *FieldType) PartialEqual(other *FieldType, unsafe bool) bool { // EvalType gets the type in evaluation. func (ft *FieldType) EvalType() EvalType { - switch ft.tp { + switch ft.GetType() { case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong, mysql.TypeBit, mysql.TypeYear: return ETInt @@ -342,7 +353,7 @@ func (ft *FieldType) EvalType() EvalType { // Hybrid checks whether a type is a hybrid type, which can represent different types of value in specific context. func (ft *FieldType) Hybrid() bool { - return ft.tp == mysql.TypeEnum || ft.tp == mysql.TypeBit || ft.tp == mysql.TypeSet + return ft.GetType() == mysql.TypeEnum || ft.GetType() == mysql.TypeBit || ft.GetType() == mysql.TypeSet } // Init initializes the FieldType data. @@ -355,10 +366,10 @@ func (ft *FieldType) Init(tp byte) { // CompactStr only considers tp/CharsetBin/flen/Deimal. // This is used for showing column type in infoschema. func (ft *FieldType) CompactStr() string { - ts := TypeToStr(ft.tp, ft.charset) + ts := TypeToStr(ft.GetType(), ft.charset) suffix := "" - defaultFlen, defaultDecimal := mysql.GetDefaultFieldLengthAndDecimal(ft.tp) + defaultFlen, defaultDecimal := mysql.GetDefaultFieldLengthAndDecimal(ft.GetType()) isDecimalNotDefault := ft.decimal != defaultDecimal && ft.decimal != 0 && ft.decimal != UnspecifiedLength // displayFlen and displayDecimal are flen and decimal values with `-1` substituted with default value. @@ -370,7 +381,7 @@ func (ft *FieldType) CompactStr() string { displayDecimal = defaultDecimal } - switch ft.tp { + switch ft.GetType() { case mysql.TypeEnum, mysql.TypeSet: // Format is ENUM ('e1', 'e2') or SET ('e1', 'e2') es := make([]string, 0, len(ft.elems)) @@ -414,8 +425,8 @@ func (ft *FieldType) CompactStr() string { func (ft *FieldType) InfoSchemaStr() string { suffix := "" if mysql.HasUnsignedFlag(ft.flag) && - ft.tp != mysql.TypeBit && - ft.tp != mysql.TypeYear { + ft.GetType() != mysql.TypeBit && + ft.GetType() != mysql.TypeYear { suffix = " unsigned" } return ft.CompactStr() + suffix @@ -431,11 +442,11 @@ func (ft *FieldType) String() string { if mysql.HasZerofillFlag(ft.flag) { strs = append(strs, "ZEROFILL") } - if mysql.HasBinaryFlag(ft.flag) && ft.tp != mysql.TypeString { + if mysql.HasBinaryFlag(ft.flag) && ft.GetType() != mysql.TypeString { strs = append(strs, "BINARY") } - if IsTypeChar(ft.tp) || IsTypeBlob(ft.tp) { + if IsTypeChar(ft.GetType()) || IsTypeBlob(ft.GetType()) { if ft.charset != "" && ft.charset != charset.CharsetBin { strs = append(strs, fmt.Sprintf("CHARACTER SET %s", ft.charset)) } @@ -449,12 +460,12 @@ func (ft *FieldType) String() string { // Restore implements Node interface. func (ft *FieldType) Restore(ctx *format.RestoreCtx) error { - ctx.WriteKeyWord(TypeToStr(ft.tp, ft.charset)) + ctx.WriteKeyWord(TypeToStr(ft.GetType(), ft.charset)) precision := UnspecifiedLength scale := UnspecifiedLength - switch ft.tp { + switch ft.GetType() { case mysql.TypeEnum, mysql.TypeSet: ctx.WritePlain("(") for i, e := range ft.elems { @@ -491,7 +502,7 @@ func (ft *FieldType) Restore(ctx *format.RestoreCtx) error { ctx.WriteKeyWord(" BINARY") } - if IsTypeChar(ft.tp) || IsTypeBlob(ft.tp) { + if IsTypeChar(ft.GetType()) || IsTypeBlob(ft.GetType()) { if ft.charset != "" && ft.charset != charset.CharsetBin { ctx.WriteKeyWord(" CHARACTER SET " + ft.charset) } @@ -519,7 +530,7 @@ func (ft *FieldType) RestoreAsCastType(ctx *format.RestoreCtx, explicitCharset b ctx.WritePlainf("(%d)", ft.flen) } if !explicitCharset { - return + break } if !skipWriteBinary && ft.flag&mysql.BinaryFlag != 0 { ctx.WriteKeyWord(" BINARY") @@ -581,7 +592,7 @@ const VarStorageLen = -1 // StorageLength is the length of stored value for the type. func (ft *FieldType) StorageLength() int { - switch ft.tp { + switch ft.GetType() { case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong, mysql.TypeDouble, mysql.TypeFloat, mysql.TypeYear, mysql.TypeDuration, mysql.TypeDate, mysql.TypeDatetime, mysql.TypeTimestamp, mysql.TypeEnum, mysql.TypeSet, @@ -599,7 +610,7 @@ func (ft *FieldType) StorageLength() int { // HasCharset indicates if a COLUMN has an associated charset. Returning false here prevents some information // statements(like `SHOW CREATE TABLE`) from attaching a CHARACTER SET clause to the column. func HasCharset(ft *FieldType) bool { - switch ft.tp { + switch ft.GetType() { case mysql.TypeVarchar, mysql.TypeString, mysql.TypeVarString, mysql.TypeBlob, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob: return !mysql.HasBinaryFlag(ft.flag) diff --git a/planner/core/expression_rewriter.go b/planner/core/expression_rewriter.go index d0ca6e6f8e4cf..ddb905dc5c06b 100644 --- a/planner/core/expression_rewriter.go +++ b/planner/core/expression_rewriter.go @@ -55,7 +55,7 @@ func evalAstExpr(sctx sessionctx.Context, expr ast.ExprNode) (types.Datum, error if val, ok := expr.(*driver.ValueExpr); ok { return val.Datum, nil } - newExpr, err := rewriteAstExpr(sctx, expr, nil, nil) + newExpr, err := rewriteAstExpr(sctx, expr, nil, nil, false) if err != nil { return types.Datum{}, err } @@ -63,13 +63,14 @@ func evalAstExpr(sctx sessionctx.Context, expr ast.ExprNode) (types.Datum, error } // rewriteAstExpr rewrites ast expression directly. -func rewriteAstExpr(sctx sessionctx.Context, expr ast.ExprNode, schema *expression.Schema, names types.NameSlice) (expression.Expression, error) { +func rewriteAstExpr(sctx sessionctx.Context, expr ast.ExprNode, schema *expression.Schema, names types.NameSlice, allowCastArray bool) (expression.Expression, error) { var is infoschema.InfoSchema // in tests, it may be null if s, ok := sctx.GetInfoSchema().(infoschema.InfoSchema); ok { is = s } b, savedBlockNames := NewPlanBuilder().Init(sctx, is, &hint.BlockHintProcessor{}) + b.allowBuildCastArray = allowCastArray fakePlan := LogicalTableDual{}.Init(sctx, 0) if schema != nil { fakePlan.schema = schema @@ -1183,6 +1184,10 @@ func (er *expressionRewriter) Leave(originInNode ast.Node) (retNode ast.Node, ok er.disableFoldCounter-- } case *ast.FuncCastExpr: + if v.Tp.IsArray() && !er.b.allowBuildCastArray { + er.err = expression.ErrNotSupportedYet.GenWithStackByArgs("Use of CAST( .. AS .. ARRAY) outside of functional index in CREATE(non-SELECT)/ALTER TABLE or in general expressions") + return retNode, false + } arg := er.ctxStack[len(er.ctxStack)-1] er.err = expression.CheckArgsNotMultiColumnRow(arg) if er.err != nil { @@ -1195,7 +1200,11 @@ func (er *expressionRewriter) Leave(originInNode ast.Node) (retNode ast.Node, ok return retNode, false } - castFunction := expression.BuildCastFunction(er.sctx, arg, v.Tp) + castFunction, err := expression.BuildCastFunctionWithCheck(er.sctx, arg, v.Tp) + if err != nil { + er.err = err + return retNode, false + } if v.Tp.EvalType() == types.ETString { castFunction.SetCoercibility(expression.CoercibilityImplicit) if v.Tp.GetCharset() == charset.CharsetASCII { diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index 9201f953bdcdc..df7a0d893e4ed 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -577,6 +577,9 @@ type PlanBuilder struct { // disableSubQueryPreprocessing indicates whether to pre-process uncorrelated sub-queries in rewriting stage. disableSubQueryPreprocessing bool + + // allowBuildCastArray indicates whether allow cast(... as ... array). + allowBuildCastArray bool } type handleColHelper struct { @@ -697,6 +700,14 @@ func (p PlanBuilderOptNoExecution) Apply(builder *PlanBuilder) { builder.disableSubQueryPreprocessing = true } +// PlanBuilderOptAllowCastArray means the plan builder should allow build cast(... as ... array). +type PlanBuilderOptAllowCastArray struct{} + +// Apply implements the interface PlanBuilderOpt. +func (p PlanBuilderOptAllowCastArray) Apply(builder *PlanBuilder) { + builder.allowBuildCastArray = true +} + // NewPlanBuilder creates a new PlanBuilder. func NewPlanBuilder(opts ...PlanBuilderOpt) *PlanBuilder { builder := &PlanBuilder{ diff --git a/types/field_type_builder.go b/types/field_type_builder.go index 7c9f3bdc3177d..81554c4585442 100644 --- a/types/field_type_builder.go +++ b/types/field_type_builder.go @@ -114,6 +114,12 @@ func (b *FieldTypeBuilder) SetElems(elems []string) *FieldTypeBuilder { return b } +// SetArray sets array of the ft +func (b *FieldTypeBuilder) SetArray(x bool) *FieldTypeBuilder { + b.ft.SetArray(x) + return b +} + // Build returns the ft func (b *FieldTypeBuilder) Build() FieldType { return b.ft From 9f4dd8039b18300d349ae8b2a655fdf45c04e3b0 Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Tue, 20 Dec 2022 13:50:54 +0800 Subject: [PATCH 3/5] planner: fix the issue that NonPrep Cache cannot work with Explain well (#40044) close pingcap/tidb#40039 --- executor/prepared.go | 2 +- planner/core/plan_cache_test.go | 31 +++++++++++++++++++++++++++++++ planner/core/plan_cache_utils.go | 24 +++++++++++++----------- planner/optimize.go | 2 +- 4 files changed, 46 insertions(+), 13 deletions(-) diff --git a/executor/prepared.go b/executor/prepared.go index a9dd9452e3c99..6a5025e0d539b 100644 --- a/executor/prepared.go +++ b/executor/prepared.go @@ -115,7 +115,7 @@ func (e *PrepareExec) Next(ctx context.Context, req *chunk.Chunk) error { return err } } - stmt, p, paramCnt, err := plannercore.GeneratePlanCacheStmtWithAST(ctx, e.ctx, stmt0) + stmt, p, paramCnt, err := plannercore.GeneratePlanCacheStmtWithAST(ctx, e.ctx, stmt0.Text(), stmt0) if err != nil { return err } diff --git a/planner/core/plan_cache_test.go b/planner/core/plan_cache_test.go index f541c441fd4f5..5c3c7c45b9702 100644 --- a/planner/core/plan_cache_test.go +++ b/planner/core/plan_cache_test.go @@ -81,6 +81,37 @@ func TestInitLRUWithSystemVar(t *testing.T) { require.NotNil(t, lru) } +func TestNonPreparedPlanCacheWithExplain(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec(`use test`) + tk.MustExec("create table t(a int)") + tk.MustExec("set tidb_enable_non_prepared_plan_cache=1") + tk.MustExec("select * from t where a=1") // cache this plan + + tk.MustQuery("explain select * from t where a=2").Check(testkit.Rows( + `Selection_8 10.00 root eq(test.t.a, 2)`, + `└─TableReader_7 10.00 root data:Selection_6`, + ` └─Selection_6 10.00 cop[tikv] eq(test.t.a, 2)`, + ` └─TableFullScan_5 10000.00 cop[tikv] table:t keep order:false, stats:pseudo`)) + tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("1")) + + tk.MustQuery("explain format=verbose select * from t where a=2").Check(testkit.Rows( + `Selection_8 10.00 169474.57 root eq(test.t.a, 2)`, + `└─TableReader_7 10.00 168975.57 root data:Selection_6`, + ` └─Selection_6 10.00 2534000.00 cop[tikv] eq(test.t.a, 2)`, + ` └─TableFullScan_5 10000.00 2035000.00 cop[tikv] table:t keep order:false, stats:pseudo`)) + tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("1")) + + tk.MustQuery("explain analyze select * from t where a=2").CheckAt([]int{0, 1, 2, 3}, [][]interface{}{ + {"Selection_8", "10.00", "0", "root"}, + {"└─TableReader_7", "10.00", "0", "root"}, + {" └─Selection_6", "10.00", "0", "cop[tikv]"}, + {" └─TableFullScan_5", "10000.00", "0", "cop[tikv]"}, + }) + tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("1")) +} + func TestNonPreparedPlanCacheBasically(t *testing.T) { store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) diff --git a/planner/core/plan_cache_utils.go b/planner/core/plan_cache_utils.go index 2b1621857b9ca..1cb66d3cdeb6a 100644 --- a/planner/core/plan_cache_utils.go +++ b/planner/core/plan_cache_utils.go @@ -64,17 +64,19 @@ func (e *paramMarkerExtractor) Leave(in ast.Node) (ast.Node, bool) { } // GeneratePlanCacheStmtWithAST generates the PlanCacheStmt structure for this AST. -func GeneratePlanCacheStmtWithAST(ctx context.Context, sctx sessionctx.Context, stmt ast.StmtNode) (*PlanCacheStmt, Plan, int, error) { +// paramSQL is the corresponding parameterized sql like 'select * from t where a?'. +// paramStmt is the Node of paramSQL. +func GeneratePlanCacheStmtWithAST(ctx context.Context, sctx sessionctx.Context, paramSQL string, paramStmt ast.StmtNode) (*PlanCacheStmt, Plan, int, error) { vars := sctx.GetSessionVars() var extractor paramMarkerExtractor - stmt.Accept(&extractor) + paramStmt.Accept(&extractor) // DDL Statements can not accept parameters - if _, ok := stmt.(ast.DDLNode); ok && len(extractor.markers) > 0 { + if _, ok := paramStmt.(ast.DDLNode); ok && len(extractor.markers) > 0 { return nil, nil, 0, ErrPrepareDDL } - switch stmt.(type) { + switch paramStmt.(type) { case *ast.LoadDataStmt, *ast.PrepareStmt, *ast.ExecuteStmt, *ast.DeallocateStmt, *ast.NonTransactionalDMLStmt: return nil, nil, 0, ErrUnsupportedPs } @@ -86,7 +88,7 @@ func GeneratePlanCacheStmtWithAST(ctx context.Context, sctx sessionctx.Context, } ret := &PreprocessorReturn{} - err := Preprocess(ctx, sctx, stmt, InPrepare, WithPreprocessorReturn(ret)) + err := Preprocess(ctx, sctx, paramStmt, InPrepare, WithPreprocessorReturn(ret)) if err != nil { return nil, nil, 0, err } @@ -103,8 +105,8 @@ func GeneratePlanCacheStmtWithAST(ctx context.Context, sctx sessionctx.Context, } prepared := &ast.Prepared{ - Stmt: stmt, - StmtType: ast.GetStmtLabel(stmt), + Stmt: paramStmt, + StmtType: ast.GetStmtLabel(paramStmt), Params: extractor.markers, SchemaVersion: ret.InfoSchema.SchemaMetaVersion(), } @@ -117,12 +119,12 @@ func GeneratePlanCacheStmtWithAST(ctx context.Context, sctx sessionctx.Context, if !vars.EnablePreparedPlanCache { prepared.UseCache = false } else { - cacheable, reason := CacheableWithCtx(sctx, stmt, ret.InfoSchema) + cacheable, reason := CacheableWithCtx(sctx, paramStmt, ret.InfoSchema) prepared.UseCache = cacheable if !cacheable { sctx.GetSessionVars().StmtCtx.AppendWarning(errors.Errorf("skip plan-cache: " + reason)) } - selectStmtNode, normalizedSQL4PC, digest4PC, err = ExtractSelectAndNormalizeDigest(stmt, vars.CurrentDB) + selectStmtNode, normalizedSQL4PC, digest4PC, err = ExtractSelectAndNormalizeDigest(paramStmt, vars.CurrentDB) if err != nil || selectStmtNode == nil { normalizedSQL4PC = "" digest4PC = "" @@ -138,7 +140,7 @@ func GeneratePlanCacheStmtWithAST(ctx context.Context, sctx sessionctx.Context, var p Plan destBuilder, _ := NewPlanBuilder().Init(sctx, ret.InfoSchema, &hint.BlockHintProcessor{}) - p, err = destBuilder.Build(ctx, stmt) + p, err = destBuilder.Build(ctx, paramStmt) if err != nil { return nil, nil, 0, err } @@ -146,7 +148,7 @@ func GeneratePlanCacheStmtWithAST(ctx context.Context, sctx sessionctx.Context, preparedObj := &PlanCacheStmt{ PreparedAst: prepared, StmtDB: vars.CurrentDB, - StmtText: stmt.Text(), + StmtText: paramSQL, VisitInfos: destBuilder.GetVisitInfo(), NormalizedSQL: normalizedSQL, SQLDigest: digest, diff --git a/planner/optimize.go b/planner/optimize.go index 3a6804d5fa319..0f37a59788a5c 100644 --- a/planner/optimize.go +++ b/planner/optimize.go @@ -85,7 +85,7 @@ func getPlanFromNonPreparedPlanCache(ctx context.Context, sctx sessionctx.Contex } val := sctx.GetSessionVars().GetNonPreparedPlanCacheStmt(paramSQL) if val == nil { - cachedStmt, _, _, err := core.GeneratePlanCacheStmtWithAST(ctx, sctx, stmt) + cachedStmt, _, _, err := core.GeneratePlanCacheStmtWithAST(ctx, sctx, paramSQL, stmt) if err != nil { return nil, nil, false, err } From 030c50604aeb2448fddb018dac5687a29b62972c Mon Sep 17 00:00:00 2001 From: wjHuang Date: Tue, 20 Dec 2022 15:02:54 +0800 Subject: [PATCH 4/5] ddl: fix waiting for wrong schema version if TiDB server restart (#39985) close pingcap/tidb#39981 --- ddl/ddl_worker.go | 38 ++++++++++++++++++++++++++++++++++---- ddl/job_table.go | 6 ++---- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/ddl/ddl_worker.go b/ddl/ddl_worker.go index 8621dcb08361c..89e515db8e1bc 100644 --- a/ddl/ddl_worker.go +++ b/ddl/ddl_worker.go @@ -536,16 +536,20 @@ func cleanMDLInfo(pool *sessionPool, jobID int64, ec *clientv3.Client) { } // checkMDLInfo checks if metadata lock info exists. It means the schema is locked by some TiDBs if exists. -func checkMDLInfo(jobID int64, pool *sessionPool) (bool, error) { - sql := fmt.Sprintf("select * from mysql.tidb_mdl_info where job_id = %d", jobID) +func checkMDLInfo(jobID int64, pool *sessionPool) (bool, int64, error) { + sql := fmt.Sprintf("select version from mysql.tidb_mdl_info where job_id = %d", jobID) sctx, _ := pool.get() defer pool.put(sctx) sess := newSession(sctx) rows, err := sess.execute(context.Background(), sql, "check-mdl-info") if err != nil { - return false, err + return false, 0, err } - return len(rows) > 0, nil + if len(rows) == 0 { + return false, 0, nil + } + ver := rows[0].GetInt64(0) + return true, ver, nil } func needUpdateRawArgs(job *model.Job, meetErr bool) bool { @@ -1377,6 +1381,32 @@ func waitSchemaChanged(ctx context.Context, d *ddlCtx, waitTime time.Duration, l zap.String("job", job.String())) } +// waitSchemaSyncedForMDL likes waitSchemaSynced, but it waits for getting the metadata lock of the latest version of this DDL. +func waitSchemaSyncedForMDL(d *ddlCtx, job *model.Job, latestSchemaVersion int64) error { + failpoint.Inject("checkDownBeforeUpdateGlobalVersion", func(val failpoint.Value) { + if val.(bool) { + if mockDDLErrOnce > 0 && mockDDLErrOnce != latestSchemaVersion { + panic("check down before update global version failed") + } else { + mockDDLErrOnce = -1 + } + } + }) + + timeStart := time.Now() + // OwnerCheckAllVersions returns only when all TiDB schemas are synced(exclude the isolated TiDB). + err := d.schemaSyncer.OwnerCheckAllVersions(context.Background(), job.ID, latestSchemaVersion) + if err != nil { + logutil.Logger(d.ctx).Info("[ddl] wait latest schema version encounter error", zap.Int64("ver", latestSchemaVersion), zap.Error(err)) + return err + } + logutil.Logger(d.ctx).Info("[ddl] wait latest schema version changed(get the metadata lock if tidb_enable_metadata_lock is true)", + zap.Int64("ver", latestSchemaVersion), + zap.Duration("take time", time.Since(timeStart)), + zap.String("job", job.String())) + return nil +} + // waitSchemaSynced handles the following situation: // If the job enters a new state, and the worker crashs when it's in the process of waiting for 2 * lease time, // Then the worker restarts quickly, we may run the job immediately again, diff --git a/ddl/job_table.go b/ddl/job_table.go index a6e19b7f7edf0..771a83b8f8264 100644 --- a/ddl/job_table.go +++ b/ddl/job_table.go @@ -237,7 +237,7 @@ func (d *ddl) delivery2worker(wk *worker, pool *workerPool, job *model.Job) { // check if this ddl job is synced to all servers. if !d.isSynced(job) || d.once.Load() { if variable.EnableMDL.Load() { - exist, err := checkMDLInfo(job.ID, d.sessPool) + exist, version, err := checkMDLInfo(job.ID, d.sessPool) if err != nil { logutil.BgLogger().Warn("[ddl] check MDL info failed", zap.Error(err), zap.String("job", job.String())) // Release the worker resource. @@ -246,10 +246,8 @@ func (d *ddl) delivery2worker(wk *worker, pool *workerPool, job *model.Job) { } else if exist { // Release the worker resource. pool.put(wk) - err = waitSchemaSynced(d.ddlCtx, job, 2*d.lease) + err = waitSchemaSyncedForMDL(d.ddlCtx, job, version) if err != nil { - logutil.BgLogger().Warn("[ddl] wait ddl job sync failed", zap.Error(err), zap.String("job", job.String())) - time.Sleep(time.Second) return } d.once.Store(false) From 4a72171ffb55b9dcf80d2dea8788ab62aff22690 Mon Sep 17 00:00:00 2001 From: Mattias Jonsson Date: Tue, 20 Dec 2022 09:44:54 +0100 Subject: [PATCH 5/5] *: Fix issue 39999, used wrong column id list for checking partitions (#40003) close pingcap/tidb#39999 --- executor/builder.go | 85 +++++++++++++----------------- executor/index_lookup_join_test.go | 20 +++---- executor/partition_table_test.go | 69 ++++++++++++++++++++++++ 3 files changed, 118 insertions(+), 56 deletions(-) diff --git a/executor/builder.go b/executor/builder.go index b33c57d3de234..d4270397eecd0 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -3511,17 +3511,39 @@ func buildIndexRangeForEachPartition(ctx sessionctx.Context, usedPartitions []ta return nextRange, nil } -func keyColumnsIncludeAllPartitionColumns(keyColumns []int, pe *tables.PartitionExpr) bool { - tmp := make(map[int]struct{}, len(keyColumns)) - for _, offset := range keyColumns { - tmp[offset] = struct{}{} +func getPartitionKeyColOffsets(keyColIDs []int64, pt table.PartitionedTable) []int { + keyColOffsets := make([]int, len(keyColIDs)) + for i, colID := range keyColIDs { + offset := -1 + for j, col := range pt.Cols() { + if colID == col.ID { + offset = j + break + } + } + if offset == -1 { + return nil + } + keyColOffsets[i] = offset + } + + pe, err := pt.(interface { + PartitionExpr() (*tables.PartitionExpr, error) + }).PartitionExpr() + if err != nil { + return nil + } + + offsetMap := make(map[int]struct{}) + for _, offset := range keyColOffsets { + offsetMap[offset] = struct{}{} } for _, offset := range pe.ColumnOffset { - if _, ok := tmp[offset]; !ok { - return false + if _, ok := offsetMap[offset]; !ok { + return nil } } - return true + return keyColOffsets } func (builder *dataReaderBuilder) prunePartitionForInnerExecutor(tbl table.Table, schema *expression.Schema, partitionInfo *plannercore.PartitionInfo, @@ -3536,15 +3558,6 @@ func (builder *dataReaderBuilder) prunePartitionForInnerExecutor(tbl table.Table return nil, false, nil, err } - // check whether can runtime prune. - type partitionExpr interface { - PartitionExpr() (*tables.PartitionExpr, error) - } - pe, err := tbl.(partitionExpr).PartitionExpr() - if err != nil { - return nil, false, nil, err - } - // recalculate key column offsets if len(lookUpContent) == 0 { return nil, false, nil, nil @@ -3552,29 +3565,9 @@ func (builder *dataReaderBuilder) prunePartitionForInnerExecutor(tbl table.Table if lookUpContent[0].keyColIDs == nil { return nil, false, nil, plannercore.ErrInternal.GenWithStack("cannot get column IDs when dynamic pruning") } - keyColOffsets := make([]int, len(lookUpContent[0].keyColIDs)) - for i, colID := range lookUpContent[0].keyColIDs { - offset := -1 - for j, col := range partitionTbl.Cols() { - if colID == col.ID { - offset = j - break - } - } - if offset == -1 { - return nil, false, nil, plannercore.ErrInternal.GenWithStack("invalid column offset when dynamic pruning") - } - keyColOffsets[i] = offset - } - - offsetMap := make(map[int]bool) - for _, offset := range keyColOffsets { - offsetMap[offset] = true - } - for _, offset := range pe.ColumnOffset { - if _, ok := offsetMap[offset]; !ok { - return condPruneResult, false, nil, nil - } + keyColOffsets := getPartitionKeyColOffsets(lookUpContent[0].keyColIDs, partitionTbl) + if len(keyColOffsets) == 0 { + return condPruneResult, false, nil, nil } locateKey := make([]types.Datum, len(partitionTbl.Cols())) @@ -4149,12 +4142,6 @@ func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Conte } tbl, _ := builder.is.TableByID(tbInfo.ID) pt := tbl.(table.PartitionedTable) - pe, err := tbl.(interface { - PartitionExpr() (*tables.PartitionExpr, error) - }).PartitionExpr() - if err != nil { - return nil, err - } partitionInfo := &v.PartitionInfo usedPartitionList, err := builder.partitionPruning(pt, partitionInfo.PruningConds, partitionInfo.PartitionNames, partitionInfo.Columns, partitionInfo.ColumnNames) if err != nil { @@ -4165,8 +4152,12 @@ func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Conte usedPartitions[p.GetPhysicalID()] = p } var kvRanges []kv.KeyRange + var keyColOffsets []int + if len(lookUpContents) > 0 { + keyColOffsets = getPartitionKeyColOffsets(lookUpContents[0].keyColIDs, pt) + } if v.IsCommonHandle { - if len(lookUpContents) > 0 && keyColumnsIncludeAllPartitionColumns(lookUpContents[0].keyCols, pe) { + if len(keyColOffsets) > 0 { locateKey := make([]types.Datum, e.Schema().Len()) kvRanges = make([]kv.KeyRange, 0, len(lookUpContents)) // lookUpContentsByPID groups lookUpContents by pid(partition) so that kv ranges for same partition can be merged. @@ -4212,7 +4203,7 @@ func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Conte handles, lookUpContents := dedupHandles(lookUpContents) - if len(lookUpContents) > 0 && keyColumnsIncludeAllPartitionColumns(lookUpContents[0].keyCols, pe) { + if len(keyColOffsets) > 0 { locateKey := make([]types.Datum, e.Schema().Len()) kvRanges = make([]kv.KeyRange, 0, len(lookUpContents)) for _, content := range lookUpContents { diff --git a/executor/index_lookup_join_test.go b/executor/index_lookup_join_test.go index 9a021568b20ee..600f052b1225e 100644 --- a/executor/index_lookup_join_test.go +++ b/executor/index_lookup_join_test.go @@ -428,17 +428,19 @@ PARTITIONS 1`) // Why does the t2.prefiller need be at least 2^32 ? If smaller the bug will not appear!?! tk.MustExec("insert into t2 values ( pow(2,32), 1, 1), ( pow(2,32)+1, 2, 0)") + tk.MustExec(`analyze table t1`) + tk.MustExec(`analyze table t2`) // Why must it be = 1 and not 2? - tk.MustQuery("explain select /* +INL_JOIN(t1,t2) */ t1.id, t1.pc from t1 where id in ( select prefiller from t2 where t2.postfiller = 1 )").Check(testkit.Rows("" + - "IndexJoin_15 10.00 root inner join, inner:TableReader_14, outer key:test.t2.prefiller, inner key:test.t1.id, equal cond:eq(test.t2.prefiller, test.t1.id)]\n" + - "[├─HashAgg_25(Build) 8.00 root group by:test.t2.prefiller, funcs:firstrow(test.t2.prefiller)->test.t2.prefiller]\n" + - "[│ └─TableReader_26 8.00 root data:HashAgg_20]\n" + - "[│ └─HashAgg_20 8.00 cop[tikv] group by:test.t2.prefiller, ]\n" + - "[│ └─Selection_24 10.00 cop[tikv] eq(test.t2.postfiller, 1)]\n" + - "[│ └─TableFullScan_23 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo]\n" + - "[└─TableReader_14(Probe) 8.00 root partition:all data:TableRangeScan_13]\n" + - "[ └─TableRangeScan_13 8.00 cop[tikv] table:t1 range: decided by [eq(test.t1.id, test.t2.prefiller)], keep order:false, stats:pseudo")) + tk.MustQuery("explain format='brief' select /* +INL_JOIN(t1,t2) */ t1.id, t1.pc from t1 where id in ( select prefiller from t2 where t2.postfiller = 1 )").Check(testkit.Rows(""+ + `IndexJoin 1.25 root inner join, inner:TableReader, outer key:test.t2.prefiller, inner key:test.t1.id, equal cond:eq(test.t2.prefiller, test.t1.id)`, + `├─HashAgg(Build) 1.00 root group by:test.t2.prefiller, funcs:firstrow(test.t2.prefiller)->test.t2.prefiller`, + `│ └─TableReader 1.00 root data:HashAgg`, + `│ └─HashAgg 1.00 cop[tikv] group by:test.t2.prefiller, `, + `│ └─Selection 1.00 cop[tikv] eq(test.t2.postfiller, 1)`, + `│ └─TableFullScan 2.00 cop[tikv] table:t2 keep order:false`, + `└─TableReader(Probe) 1.00 root partition:all data:TableRangeScan`, + ` └─TableRangeScan 1.00 cop[tikv] table:t1 range: decided by [eq(test.t1.id, test.t2.prefiller)], keep order:false, stats:pseudo`)) tk.MustQuery("show warnings").Check(testkit.Rows()) // without fix it fails with: "runtime error: index out of range [0] with length 0" tk.MustQuery("select /* +INL_JOIN(t1,t2) */ t1.id, t1.pc from t1 where id in ( select prefiller from t2 where t2.postfiller = 1 )").Check(testkit.Rows()) diff --git a/executor/partition_table_test.go b/executor/partition_table_test.go index 15d2c2872ca9c..5696b56f6f730 100644 --- a/executor/partition_table_test.go +++ b/executor/partition_table_test.go @@ -3831,3 +3831,72 @@ func TestIssue21732(t *testing.T) { }) } } + +func TestIssue39999(t *testing.T) { + store := testkit.CreateMockStore(t) + + tk := testkit.NewTestKit(t, store) + + tk.MustExec(`create schema test39999`) + tk.MustExec(`use test39999`) + tk.MustExec(`drop table if exists c, t`) + tk.MustExec("CREATE TABLE `c` (" + + "`serial_id` varchar(24)," + + "`occur_trade_date` date," + + "`txt_account_id` varchar(24)," + + "`capital_sub_class` varchar(10)," + + "`occur_amount` decimal(16,2)," + + "`broker` varchar(10)," + + "PRIMARY KEY (`txt_account_id`,`occur_trade_date`,`serial_id`) /*T![clustered_index] CLUSTERED */," + + "KEY `idx_serial_id` (`serial_id`)" + + ") ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci " + + "PARTITION BY RANGE COLUMNS(`serial_id`) (" + + "PARTITION `p202209` VALUES LESS THAN ('20221001')," + + "PARTITION `p202210` VALUES LESS THAN ('20221101')," + + "PARTITION `p202211` VALUES LESS THAN ('20221201')" + + ")") + + tk.MustExec("CREATE TABLE `t` ( " + + "`txn_account_id` varchar(24), " + + "`account_id` varchar(32), " + + "`broker` varchar(10), " + + "PRIMARY KEY (`txn_account_id`) /*T![clustered_index] CLUSTERED */ " + + ") ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci") + + tk.MustExec("INSERT INTO `c` (serial_id, txt_account_id, capital_sub_class, occur_trade_date, occur_amount, broker) VALUES ('2022111700196920','04482786','CUST','2022-11-17',-2.01,'0009')") + tk.MustExec("INSERT INTO `t` VALUES ('04482786','1142927','0009')") + + tk.MustExec(`set tidb_partition_prune_mode='dynamic'`) + tk.MustExec(`analyze table c`) + tk.MustExec(`analyze table t`) + query := `select + /*+ inl_join(c) */ + c.occur_amount +from + c + join t on c.txt_account_id = t.txn_account_id + and t.broker = '0009' + and c.occur_trade_date = '2022-11-17'` + tk.MustQuery("explain " + query).Check(testkit.Rows(""+ + "IndexJoin_22 1.00 root inner join, inner:TableReader_21, outer key:test39999.t.txn_account_id, inner key:test39999.c.txt_account_id, equal cond:eq(test39999.t.txn_account_id, test39999.c.txt_account_id)", + "├─TableReader_27(Build) 1.00 root data:Selection_26", + "│ └─Selection_26 1.00 cop[tikv] eq(test39999.t.broker, \"0009\")", + "│ └─TableFullScan_25 1.00 cop[tikv] table:t keep order:false", + "└─TableReader_21(Probe) 1.00 root partition:all data:Selection_20", + " └─Selection_20 1.00 cop[tikv] eq(test39999.c.occur_trade_date, 2022-11-17 00:00:00.000000)", + " └─TableRangeScan_19 1.00 cop[tikv] table:c range: decided by [eq(test39999.c.txt_account_id, test39999.t.txn_account_id) eq(test39999.c.occur_trade_date, 2022-11-17 00:00:00.000000)], keep order:false")) + tk.MustQuery(query).Check(testkit.Rows("-2.01")) + + // Add the missing partition key part. + tk.MustExec(`alter table t add column serial_id varchar(24) default '2022111700196920'`) + query += ` and c.serial_id = t.serial_id` + tk.MustQuery(query).Check(testkit.Rows("-2.01")) + tk.MustQuery("explain " + query).Check(testkit.Rows(""+ + `IndexJoin_20 0.80 root inner join, inner:TableReader_19, outer key:test39999.t.txn_account_id, test39999.t.serial_id, inner key:test39999.c.txt_account_id, test39999.c.serial_id, equal cond:eq(test39999.t.serial_id, test39999.c.serial_id), eq(test39999.t.txn_account_id, test39999.c.txt_account_id)`, + `├─TableReader_25(Build) 0.80 root data:Selection_24`, + `│ └─Selection_24 0.80 cop[tikv] eq(test39999.t.broker, "0009"), not(isnull(test39999.t.serial_id))`, + `│ └─TableFullScan_23 1.00 cop[tikv] table:t keep order:false`, + `└─TableReader_19(Probe) 0.80 root partition:all data:Selection_18`, + ` └─Selection_18 0.80 cop[tikv] eq(test39999.c.occur_trade_date, 2022-11-17 00:00:00.000000)`, + ` └─TableRangeScan_17 0.80 cop[tikv] table:c range: decided by [eq(test39999.c.txt_account_id, test39999.t.txn_account_id) eq(test39999.c.serial_id, test39999.t.serial_id) eq(test39999.c.occur_trade_date, 2022-11-17 00:00:00.000000)], keep order:false`)) +}