From 465af00b46fe64f46fbc58b64b5535e888a1f53d Mon Sep 17 00:00:00 2001 From: lysu Date: Thu, 6 Feb 2020 16:40:26 +0800 Subject: [PATCH 1/5] *: use new registrable terror (#14606) --- ddl/ddl.go | 103 ------------------------ ddl/ddl_worker.go | 2 +- distsql/select_result.go | 4 +- distsql/stream.go | 2 +- domain/domain.go | 9 --- executor/errors.go | 30 ------- executor/executor_test.go | 2 +- expression/errors.go | 26 ------ expression/integration_test.go | 2 +- expression/scalar_function.go | 7 +- go.mod | 10 +-- go.sum | 24 +++--- infoschema/infoschema.go | 28 ------- kv/error.go | 17 ---- meta/autoid/errors.go | 13 --- meta/meta.go | 10 --- planner/core/errors.go | 66 --------------- plugin/errors.go | 33 ++------ privilege/privileges/errors.go | 12 +-- privilege/privileges/privileges.go | 4 +- privilege/privileges/privileges_test.go | 4 +- server/server.go | 12 --- session/tidb.go | 7 -- sessionctx/variable/sysvar.go | 18 ----- store/tikv/backoff.go | 4 +- store/tikv/error.go | 25 ++---- structure/structure.go | 11 --- table/table.go | 23 ------ tablecodec/tablecodec.go | 9 --- types/errors.go | 31 ------- types/json/constants.go | 13 --- util/admin/admin.go | 11 --- util/memory/action.go | 7 -- 33 files changed, 52 insertions(+), 527 deletions(-) diff --git a/ddl/ddl.go b/ddl/ddl.go index 9aa4d2709f15c..ed4269226d14f 100644 --- a/ddl/ddl.go +++ b/ddl/ddl.go @@ -665,106 +665,3 @@ func (d *ddl) GetHook() Callback { return d.mu.hook } - -func init() { - ddlMySQLErrCodes := map[terror.ErrCode]uint16{ - mysql.ErrAlterOperationNotSupportedReason: mysql.ErrAlterOperationNotSupportedReason, - mysql.ErrBadField: mysql.ErrBadField, - mysql.ErrBadNull: mysql.ErrBadNull, - mysql.ErrBlobCantHaveDefault: mysql.ErrBlobCantHaveDefault, - mysql.ErrBlobKeyWithoutLength: mysql.ErrBlobKeyWithoutLength, - mysql.ErrCancelledDDLJob: mysql.ErrCancelledDDLJob, - mysql.ErrCantDecodeIndex: mysql.ErrCantDecodeIndex, - mysql.ErrCantDropFieldOrKey: mysql.ErrCantDropFieldOrKey, - mysql.ErrCantRemoveAllFields: mysql.ErrCantRemoveAllFields, - mysql.ErrCoalesceOnlyOnHashPartition: mysql.ErrCoalesceOnlyOnHashPartition, - mysql.ErrCollationCharsetMismatch: mysql.ErrCollationCharsetMismatch, - mysql.ErrConflictingDeclarations: mysql.ErrConflictingDeclarations, - mysql.ErrDependentByGeneratedColumn: mysql.ErrDependentByGeneratedColumn, - mysql.ErrDropLastPartition: mysql.ErrDropLastPartition, - mysql.ErrDropPartitionNonExistent: mysql.ErrDropPartitionNonExistent, - mysql.ErrDupKeyName: mysql.ErrDupKeyName, - mysql.ErrErrorOnRename: mysql.ErrErrorOnRename, - mysql.ErrFieldNotFoundPart: mysql.ErrFieldNotFoundPart, - mysql.ErrFieldTypeNotAllowedAsPartitionField: mysql.ErrFieldTypeNotAllowedAsPartitionField, - mysql.ErrFileNotFound: mysql.ErrFileNotFound, - mysql.ErrFunctionalIndexPrimaryKey: mysql.ErrFunctionalIndexPrimaryKey, - mysql.ErrGeneratedColumnFunctionIsNotAllowed: mysql.ErrGeneratedColumnFunctionIsNotAllowed, - mysql.ErrGeneratedColumnNonPrior: mysql.ErrGeneratedColumnNonPrior, - mysql.ErrGeneratedColumnRefAutoInc: mysql.ErrGeneratedColumnRefAutoInc, - mysql.ErrInvalidAutoRandom: mysql.ErrInvalidAutoRandom, - mysql.ErrInvalidDDLJob: mysql.ErrInvalidDDLJob, - mysql.ErrInvalidDDLState: mysql.ErrInvalidDDLState, - mysql.ErrInvalidDDLWorker: mysql.ErrInvalidDDLWorker, - mysql.ErrInvalidDefault: mysql.ErrInvalidDefault, - mysql.ErrInvalidGroupFuncUse: mysql.ErrInvalidGroupFuncUse, - mysql.ErrInvalidDDLJobFlag: mysql.ErrInvalidDDLJobFlag, - mysql.ErrInvalidDDLJobVersion: mysql.ErrInvalidDDLJobVersion, - mysql.ErrInvalidOnUpdate: mysql.ErrInvalidOnUpdate, - mysql.ErrInvalidSplitRegionRanges: mysql.ErrInvalidSplitRegionRanges, - mysql.ErrInvalidStoreVersion: mysql.ErrInvalidStoreVersion, - mysql.ErrInvalidUseOfNull: mysql.ErrInvalidUseOfNull, - mysql.ErrJSONUsedAsKey: mysql.ErrJSONUsedAsKey, - mysql.ErrJSONDocumentNULLKey: mysql.ErrJSONDocumentNULLKey, - mysql.ErrKeyColumnDoesNotExits: mysql.ErrKeyColumnDoesNotExits, - mysql.ErrLockWaitTimeout: mysql.ErrLockWaitTimeout, - mysql.ErrNoParts: mysql.ErrNoParts, - mysql.ErrNotOwner: mysql.ErrNotOwner, - mysql.ErrOnlyOnRangeListPartition: mysql.ErrOnlyOnRangeListPartition, - mysql.ErrPartitionColumnList: mysql.ErrPartitionColumnList, - mysql.ErrPartitionFuncNotAllowed: mysql.ErrPartitionFuncNotAllowed, - mysql.ErrPartitionFunctionIsNotAllowed: mysql.ErrPartitionFunctionIsNotAllowed, - mysql.ErrPartitionMaxvalue: mysql.ErrPartitionMaxvalue, - mysql.ErrPartitionMgmtOnNonpartitioned: mysql.ErrPartitionMgmtOnNonpartitioned, - mysql.ErrPartitionRequiresValues: mysql.ErrPartitionRequiresValues, - mysql.ErrPartitionWrongNoPart: mysql.ErrPartitionWrongNoPart, - mysql.ErrPartitionWrongNoSubpart: mysql.ErrPartitionWrongNoSubpart, - mysql.ErrPartitionWrongValues: mysql.ErrPartitionWrongValues, - mysql.ErrPartitionsMustBeDefined: mysql.ErrPartitionsMustBeDefined, - mysql.ErrPrimaryCantHaveNull: mysql.ErrPrimaryCantHaveNull, - mysql.ErrRangeNotIncreasing: mysql.ErrRangeNotIncreasing, - mysql.ErrRowSinglePartitionField: mysql.ErrRowSinglePartitionField, - mysql.ErrSameNamePartition: mysql.ErrSameNamePartition, - mysql.ErrSubpartition: mysql.ErrSubpartition, - mysql.ErrSystemVersioningWrongPartitions: mysql.ErrSystemVersioningWrongPartitions, - mysql.ErrTableCantHandleFt: mysql.ErrTableCantHandleFt, - mysql.ErrTableMustHaveColumns: mysql.ErrTableMustHaveColumns, - mysql.ErrTooLongIdent: mysql.ErrTooLongIdent, - mysql.ErrTooLongIndexComment: mysql.ErrTooLongIndexComment, - mysql.ErrTooLongKey: mysql.ErrTooLongKey, - mysql.ErrTooManyFields: mysql.ErrTooManyFields, - mysql.ErrTooManyPartitions: mysql.ErrTooManyPartitions, - mysql.ErrTooManyValues: mysql.ErrTooManyValues, - mysql.ErrUniqueKeyNeedAllFieldsInPf: mysql.ErrUniqueKeyNeedAllFieldsInPf, - mysql.ErrUnknownCharacterSet: mysql.ErrUnknownCharacterSet, - mysql.ErrUnknownCollation: mysql.ErrUnknownCollation, - mysql.ErrUnknownPartition: mysql.ErrUnknownPartition, - mysql.ErrUnsupportedDDLOperation: mysql.ErrUnsupportedDDLOperation, - mysql.ErrUnsupportedOnGeneratedColumn: mysql.ErrUnsupportedOnGeneratedColumn, - mysql.ErrViewWrongList: mysql.ErrViewWrongList, - mysql.ErrWrongColumnName: mysql.ErrWrongColumnName, - mysql.ErrWrongDBName: mysql.ErrWrongDBName, - mysql.ErrWrongExprInPartitionFunc: mysql.ErrWrongExprInPartitionFunc, - mysql.ErrWrongFKOptionForGeneratedColumn: mysql.ErrWrongFKOptionForGeneratedColumn, - mysql.ErrWrongKeyColumn: mysql.ErrWrongKeyColumn, - mysql.ErrWrongNameForIndex: mysql.ErrWrongNameForIndex, - mysql.ErrWrongObject: mysql.ErrWrongObject, - mysql.ErrWrongPartitionTypeExpectedSystemTime: mysql.ErrWrongPartitionTypeExpectedSystemTime, - mysql.ErrWrongSubKey: mysql.ErrWrongSubKey, - mysql.ErrWrongTableName: mysql.ErrWrongTableName, - mysql.ErrWrongTypeColumnValue: mysql.ErrWrongTypeColumnValue, - mysql.WarnDataTruncated: mysql.WarnDataTruncated, - mysql.ErrFunctionalIndexOnField: mysql.ErrFunctionalIndexOnField, - mysql.ErrFkColumnCannotDrop: mysql.ErrFkColumnCannotDrop, - mysql.ErrFKIncompatibleColumns: mysql.ErrFKIncompatibleColumns, - mysql.ErrSequenceRunOut: mysql.ErrSequenceRunOut, - mysql.ErrSequenceInvalidData: mysql.ErrSequenceInvalidData, - mysql.ErrSequenceAccessFail: mysql.ErrSequenceAccessFail, - mysql.ErrNotSequence: mysql.ErrNotSequence, - mysql.ErrUnknownSequence: mysql.ErrUnknownSequence, - mysql.ErrWrongInsertIntoSequence: mysql.ErrWrongInsertIntoSequence, - mysql.ErrSequenceInvalidTableStructure: mysql.ErrSequenceInvalidTableStructure, - mysql.ErrSequenceUnsupportedTableOption: mysql.ErrSequenceUnsupportedTableOption, - } - terror.ErrClassToMySQLCodes[terror.ClassDDL] = ddlMySQLErrCodes -} diff --git a/ddl/ddl_worker.go b/ddl/ddl_worker.go index fb9cbd97aef0f..be6c824124330 100644 --- a/ddl/ddl_worker.go +++ b/ddl/ddl_worker.go @@ -633,7 +633,7 @@ func toTError(err error) *terror.Error { } // TODO: Add the error code. - return terror.ClassDDL.New(terror.CodeUnknown, err.Error()) + return terror.ClassDDL.Synthesize(terror.CodeUnknown, err.Error()) } // waitSchemaChanged waits for the completion of updating all servers' schema. In order to make sure that happens, diff --git a/distsql/select_result.go b/distsql/select_result.go index 33ff4fec7b37a..88d43e873d851 100644 --- a/distsql/select_result.go +++ b/distsql/select_result.go @@ -114,11 +114,11 @@ func (r *selectResult) fetchResp(ctx context.Context) error { r.selectRespSize = r.selectResp.Size() r.memConsume(int64(r.selectRespSize)) if err := r.selectResp.Error; err != nil { - return terror.ClassTiKV.New(terror.ErrCode(err.Code), err.Msg) + return terror.ClassTiKV.Synthesize(terror.ErrCode(err.Code), err.Msg) } sc := r.ctx.GetSessionVars().StmtCtx for _, warning := range r.selectResp.Warnings { - sc.AppendWarning(terror.ClassTiKV.New(terror.ErrCode(warning.Code), warning.Msg)) + sc.AppendWarning(terror.ClassTiKV.Synthesize(terror.ErrCode(warning.Code), warning.Msg)) } r.updateCopRuntimeStats(resultSubset.GetExecDetails(), resultSubset.RespTime()) r.feedback.Update(resultSubset.GetStartKey(), r.selectResp.OutputCounts) diff --git a/distsql/stream.go b/distsql/stream.go index 42ed3506e09e4..6a100c2d8cfe8 100644 --- a/distsql/stream.go +++ b/distsql/stream.go @@ -89,7 +89,7 @@ func (r *streamResult) readDataFromResponse(ctx context.Context, resp kv.Respons return false, errors.Errorf("stream response error: [%d]%s\n", stream.Error.Code, stream.Error.Msg) } for _, warning := range stream.Warnings { - r.ctx.GetSessionVars().StmtCtx.AppendWarning(terror.ClassTiKV.New(terror.ErrCode(warning.Code), warning.Msg)) + r.ctx.GetSessionVars().StmtCtx.AppendWarning(terror.ClassTiKV.Synthesize(terror.ErrCode(warning.Code), warning.Msg)) } err = result.Unmarshal(stream.Data) diff --git a/domain/domain.go b/domain/domain.go index 39df1fdeaafc6..b663164c87be0 100644 --- a/domain/domain.go +++ b/domain/domain.go @@ -1134,12 +1134,3 @@ var ( ErrInfoSchemaChanged = terror.ClassDomain.New(mysql.ErrInfoSchemaChanged, mysql.MySQLErrName[mysql.ErrInfoSchemaChanged]+". "+kv.TxnRetryableMark) ) - -func init() { - // Map error codes to mysql error codes. - domainMySQLErrCodes := map[terror.ErrCode]uint16{ - mysql.ErrInfoSchemaExpired: mysql.ErrInfoSchemaExpired, - mysql.ErrInfoSchemaChanged: mysql.ErrInfoSchemaChanged, - } - terror.ErrClassToMySQLCodes[terror.ClassDomain] = domainMySQLErrCodes -} diff --git a/executor/errors.go b/executor/errors.go index cac87c2716872..801f379d1d78d 100644 --- a/executor/errors.go +++ b/executor/errors.go @@ -43,33 +43,3 @@ var ( ErrDeadlock = terror.ClassExecutor.New(mysql.ErrLockDeadlock, mysql.MySQLErrName[mysql.ErrLockDeadlock]) ErrQueryInterrupted = terror.ClassExecutor.New(mysql.ErrQueryInterrupted, mysql.MySQLErrName[mysql.ErrQueryInterrupted]) ) - -func init() { - // Map error codes to mysql error codes. - tableMySQLErrCodes := map[terror.ErrCode]uint16{ - mysql.ErrGetStartTS: mysql.ErrGetStartTS, - mysql.ErrUnknownPlan: mysql.ErrUnknownPlan, - mysql.ErrPrepareMulti: mysql.ErrPrepareMulti, - mysql.ErrPrepareDDL: mysql.ErrPrepareDDL, - mysql.ErrResultIsEmpty: mysql.ErrResultIsEmpty, - mysql.ErrBuildExecutor: mysql.ErrBuildExecutor, - mysql.ErrBatchInsertFail: mysql.ErrBatchInsertFail, - - mysql.ErrCantCreateUserWithGrant: mysql.ErrCantCreateUserWithGrant, - mysql.ErrPasswordNoMatch: mysql.ErrPasswordNoMatch, - mysql.ErrCannotUser: mysql.ErrCannotUser, - mysql.ErrPasswordFormat: mysql.ErrPasswordFormat, - mysql.ErrCantChangeTxCharacteristics: mysql.ErrCantChangeTxCharacteristics, - mysql.ErrPsManyParam: mysql.ErrPsManyParam, - mysql.ErrAdminCheckTable: mysql.ErrAdminCheckTable, - mysql.ErrDBaccessDenied: mysql.ErrDBaccessDenied, - mysql.ErrTableaccessDenied: mysql.ErrTableaccessDenied, - mysql.ErrBadDB: mysql.ErrBadDB, - mysql.ErrWrongObject: mysql.ErrWrongObject, - mysql.ErrRoleNotGranted: mysql.ErrRoleNotGranted, - mysql.ErrLockDeadlock: mysql.ErrLockDeadlock, - mysql.ErrQueryInterrupted: mysql.ErrQueryInterrupted, - mysql.ErrWrongValueCountOnRow: mysql.ErrWrongValueCountOnRow, - } - terror.ErrClassToMySQLCodes[terror.ClassExecutor] = tableMySQLErrCodes -} diff --git a/executor/executor_test.go b/executor/executor_test.go index e908d34f89b6b..909b84a207cec 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -3447,7 +3447,7 @@ func (s *testSuite) TestCoprocessorStreamingWarning(c *C) { result := tk.MustQuery("select * from t where a/0 > 1") result.Check(testkit.Rows()) - tk.MustQuery("show warnings").Check(testutil.RowsWithSep("|", "Warning|1105|Division by 0")) + tk.MustQuery("show warnings").Check(testutil.RowsWithSep("|", "Warning|1365|Division by 0")) } func (s *testSuite3) TestYearTypeDeleteIndex(c *C) { diff --git a/expression/errors.go b/expression/errors.go index 5db1acbd0b494..970c88dfa71f2 100644 --- a/expression/errors.go +++ b/expression/errors.go @@ -47,32 +47,6 @@ var ( errNonUniq = terror.ClassExpression.New(mysql.ErrNonUniq, mysql.MySQLErrName[mysql.ErrNonUniq]) ) -func init() { - expressionMySQLErrCodes := map[terror.ErrCode]uint16{ - mysql.ErrWrongParamcountToNativeFct: mysql.ErrWrongParamcountToNativeFct, - mysql.ErrDivisionByZero: mysql.ErrDivisionByZero, - mysql.ErrSpDoesNotExist: mysql.ErrSpDoesNotExist, - mysql.ErrNotSupportedYet: mysql.ErrNotSupportedYet, - mysql.ErrZlibZData: mysql.ErrZlibZData, - mysql.ErrZlibZBuf: mysql.ErrZlibZBuf, - mysql.ErrWrongArguments: mysql.ErrWrongArguments, - mysql.ErrUnknownCharacterSet: mysql.ErrUnknownCharacterSet, - mysql.ErrInvalidDefault: mysql.ErrInvalidDefault, - mysql.ErrWarnDeprecatedSyntaxNoReplacement: mysql.ErrWarnDeprecatedSyntaxNoReplacement, - mysql.ErrOperandColumns: mysql.ErrOperandColumns, - mysql.ErrCutValueGroupConcat: mysql.ErrCutValueGroupConcat, - mysql.ErrRegexp: mysql.ErrRegexp, - mysql.ErrWarnAllowedPacketOverflowed: mysql.ErrWarnAllowedPacketOverflowed, - mysql.WarnOptionIgnored: mysql.WarnOptionIgnored, - mysql.ErrTruncatedWrongValue: mysql.ErrTruncatedWrongValue, - mysql.ErrUnknownLocale: mysql.ErrUnknownLocale, - mysql.ErrBadField: mysql.ErrBadField, - mysql.ErrNonUniq: mysql.ErrNonUniq, - mysql.ErrIncorrectType: mysql.ErrIncorrectType, - } - terror.ErrClassToMySQLCodes[terror.ClassExpression] = expressionMySQLErrCodes -} - // handleInvalidTimeError reports error or warning depend on the context. func handleInvalidTimeError(ctx sessionctx.Context, err error) error { if err == nil || !(types.ErrWrongValue.Equal(err) || diff --git a/expression/integration_test.go b/expression/integration_test.go index a7d6274f76dc6..6ba5dfeb0ff15 100755 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -3250,7 +3250,7 @@ func (s *testIntegrationSuite) TestArithmeticBuiltin(c *C) { tk.MustExec("insert into t value(1.2)") result = tk.MustQuery("select * from t where a/0 > 1") result.Check(testkit.Rows()) - tk.MustQuery("show warnings").Check(testutil.RowsWithSep("|", "Warning|1105|Division by 0")) + tk.MustQuery("show warnings").Check(testutil.RowsWithSep("|", "Warning|1365|Division by 0")) tk.MustExec("USE test;") tk.MustExec("DROP TABLE IF EXISTS t;") diff --git a/expression/scalar_function.go b/expression/scalar_function.go index c0d7b104ebb3e..8c81550368d12 100755 --- a/expression/scalar_function.go +++ b/expression/scalar_function.go @@ -31,6 +31,11 @@ import ( "github.com/pingcap/tidb/util/hack" ) +// error definitions. +var ( + ErrNoDB = terror.ClassOptimizer.New(mysql.ErrNoDB, mysql.MySQLErrName[mysql.ErrNoDB]) +) + // ScalarFunction is the function that returns a value. type ScalarFunction struct { FuncName model.CIStr @@ -171,7 +176,7 @@ func newFunctionImpl(ctx sessionctx.Context, fold bool, funcName string, retType if !ok { db := ctx.GetSessionVars().CurrentDB if db == "" { - return nil, terror.ClassOptimizer.New(mysql.ErrNoDB, mysql.MySQLErrName[mysql.ErrNoDB]) + return nil, ErrNoDB } return nil, errFunctionNotExists.GenWithStackByArgs("FUNCTION", db+"."+funcName) diff --git a/go.mod b/go.mod index b6288f53bb111..2bfdc8bdc1bce 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/go-sql-driver/mysql v0.0.0-20170715192408-3955978caca4 github.com/gogo/protobuf v1.2.1 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect - github.com/golang/protobuf v1.3.2 + github.com/golang/protobuf v1.3.3 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db github.com/google/btree v1.0.0 github.com/google/pprof v0.0.0-20190930153522-6ce02741cba3 @@ -38,11 +38,11 @@ require ( github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 github.com/pingcap/kvproto v0.0.0-20200108025604-a4dc183d2af5 github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd - github.com/pingcap/parser v0.0.0-20200120100653-1d87b3907217 + github.com/pingcap/parser v0.0.0-20200206062121-701f4ad4e3c8 github.com/pingcap/pd v1.1.0-beta.0.20191219054547-4d65bbefbc6d github.com/pingcap/sysutil v0.0.0-20191216090214-5f9620d22b3b github.com/pingcap/tidb-tools v3.0.6-0.20191106033616-90632dda3863+incompatible - github.com/pingcap/tipb v0.0.0-20200103084511-1d37e605f65d + github.com/pingcap/tipb v0.0.0-20200201101609-1a2e9c441455 github.com/prometheus/client_golang v1.0.0 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 github.com/prometheus/common v0.4.1 @@ -62,12 +62,12 @@ require ( go.uber.org/automaxprocs v1.2.0 go.uber.org/zap v1.13.0 golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 // indirect - golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect + golang.org/x/lint v0.0.0-20200130185559-910be7a94367 // indirect golang.org/x/net v0.0.0-20190909003024-a7b16738d86b golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 golang.org/x/text v0.3.2 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect - golang.org/x/tools v0.0.0-20200119215504-eb0d8dd85bcc + golang.org/x/tools v0.0.0-20200206050830-dd0d5d485177 google.golang.org/genproto v0.0.0-20190905072037-92dd089d5514 // indirect google.golang.org/grpc v1.25.1 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect diff --git a/go.sum b/go.sum index 4cc0e9f9a9d56..3678f1fdcd04a 100644 --- a/go.sum +++ b/go.sum @@ -92,6 +92,8 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -207,8 +209,8 @@ github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9 h1:AJD9pZYm72vMgPcQDww github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd h1:CV3VsP3Z02MVtdpTMfEgRJ4T9NGgGTxdHpJerent7rM= github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= -github.com/pingcap/parser v0.0.0-20200120100653-1d87b3907217 h1:UtieYveNGV84dIdb01UIXMQzGIyGLRiAoGXgLe9rJws= -github.com/pingcap/parser v0.0.0-20200120100653-1d87b3907217/go.mod h1:9v0Edh8IbgjGYW2ArJr19E+bvL8zKahsFp+ixWeId+4= +github.com/pingcap/parser v0.0.0-20200206062121-701f4ad4e3c8 h1:GeqQjgIzW6WjkBu7B6a4LZ207JZNhmUS42/GjtLf9Zw= +github.com/pingcap/parser v0.0.0-20200206062121-701f4ad4e3c8/go.mod h1:9v0Edh8IbgjGYW2ArJr19E+bvL8zKahsFp+ixWeId+4= github.com/pingcap/pd v1.1.0-beta.0.20191219054547-4d65bbefbc6d h1:Ui80aiLTyd0EZD56o2tjFRYpHfhazBjtBdKeR8UoTFY= github.com/pingcap/pd v1.1.0-beta.0.20191219054547-4d65bbefbc6d/go.mod h1:CML+b1JVjN+VbDijaIcUSmuPgpDjXEY7UiOx5yDP8eE= github.com/pingcap/sysutil v0.0.0-20191216090214-5f9620d22b3b h1:EEyo/SCRswLGuSk+7SB86Ak1p8bS6HL1Mi4Dhyuv6zg= @@ -216,8 +218,8 @@ github.com/pingcap/sysutil v0.0.0-20191216090214-5f9620d22b3b/go.mod h1:EB/852NM github.com/pingcap/tidb-tools v3.0.6-0.20191106033616-90632dda3863+incompatible h1:H1jg0aDWz2SLRh3hNBo2HFtnuHtudIUvBumU7syRkic= github.com/pingcap/tidb-tools v3.0.6-0.20191106033616-90632dda3863+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= -github.com/pingcap/tipb v0.0.0-20200103084511-1d37e605f65d h1:ohGnm9xZ7pIysk7quOC7lZa8kOm9Pl5TMyjBThXqy2U= -github.com/pingcap/tipb v0.0.0-20200103084511-1d37e605f65d/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= +github.com/pingcap/tipb v0.0.0-20200201101609-1a2e9c441455 h1:Jh9k3RIOTJ/YvODLg2zcCmGaQg1wEt2iFh1vYSEZW5Q= +github.com/pingcap/tipb v0.0.0-20200201101609-1a2e9c441455/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -332,9 +334,10 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367 h1:0IiAsCRByjO2QjX7ZPkw5oU9x+n1YqRL802rjC0c3Aw= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee h1:WG0RUwxtNT4qqaXX3DPA8zHFNm/D9xaBpxzHt1WcA/E= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -383,12 +386,11 @@ golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191107010934-f79515f33823/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200114235610-7ae403b6b589 h1:rjUrONFu4kLchcZTfp3/96bR8bW8dIa8uz3cR5n0cgM= -golang.org/x/tools v0.0.0-20200114235610-7ae403b6b589/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200119215504-eb0d8dd85bcc h1:ZA7KFRdqWZkBr0/YbHm1h08vDJ5gQdjVG/8L153z5c4= -golang.org/x/tools v0.0.0-20200119215504-eb0d8dd85bcc/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200206050830-dd0d5d485177 h1:E2vxBajJgSA3TcJhDGTh/kP3VnsvXKl9jSijv+h7svQ= +golang.org/x/tools v0.0.0-20200206050830-dd0d5d485177/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= diff --git a/infoschema/infoschema.go b/infoschema/infoschema.go index 2ea7ca5708c3f..c121097a90358 100644 --- a/infoschema/infoschema.go +++ b/infoschema/infoschema.go @@ -320,34 +320,6 @@ func (h *Handle) EmptyClone() *Handle { } func init() { - schemaMySQLErrCodes := map[terror.ErrCode]uint16{ - mysql.ErrDBCreateExists: mysql.ErrDBCreateExists, - mysql.ErrDBDropExists: mysql.ErrDBDropExists, - mysql.ErrAccessDenied: mysql.ErrAccessDenied, - mysql.ErrBadDB: mysql.ErrBadDB, - mysql.ErrTableExists: mysql.ErrTableExists, - mysql.ErrBadTable: mysql.ErrBadTable, - mysql.ErrBadField: mysql.ErrBadField, - mysql.ErrDupFieldName: mysql.ErrDupFieldName, - mysql.ErrDupKeyName: mysql.ErrDupKeyName, - mysql.ErrNonuniqTable: mysql.ErrNonuniqTable, - mysql.ErrMultiplePriKey: mysql.ErrMultiplePriKey, - mysql.ErrTooManyKeyParts: mysql.ErrTooManyKeyParts, - mysql.ErrCantDropFieldOrKey: mysql.ErrCantDropFieldOrKey, - mysql.ErrTableNotLockedForWrite: mysql.ErrTableNotLockedForWrite, - mysql.ErrTableNotLocked: mysql.ErrTableNotLocked, - mysql.ErrNoSuchTable: mysql.ErrNoSuchTable, - mysql.ErrKeyDoesNotExist: mysql.ErrKeyDoesNotExist, - mysql.ErrCannotAddForeign: mysql.ErrCannotAddForeign, - mysql.ErrWrongFkDef: mysql.ErrWrongFkDef, - mysql.ErrDupIndex: mysql.ErrDupIndex, - mysql.ErrBadUser: mysql.ErrBadUser, - mysql.ErrUserAlreadyExists: mysql.ErrUserAlreadyExists, - mysql.ErrTableLocked: mysql.ErrTableLocked, - mysql.ErrUnknownSequence: mysql.ErrUnknownSequence, - } - terror.ErrClassToMySQLCodes[terror.ClassSchema] = schemaMySQLErrCodes - // Initialize the information shema database and register the driver to `drivers` dbID := autoid.InformationSchemaDBID infoSchemaTables := make([]*model.TableInfo, 0, len(tableNameToColumns)) diff --git a/kv/error.go b/kv/error.go index 35a9ae8ccc2ac..2f1fc5220e7e9 100644 --- a/kv/error.go +++ b/kv/error.go @@ -50,23 +50,6 @@ var ( mysql.MySQLErrName[mysql.ErrWriteConflictInTiDB]+" "+TxnRetryableMark) ) -func init() { - kvMySQLErrCodes := map[terror.ErrCode]uint16{ - mysql.ErrNotExist: mysql.ErrNotExist, - mysql.ErrDupEntry: mysql.ErrDupEntry, - mysql.ErrTooBigRowsize: mysql.ErrTooBigRowsize, - mysql.ErrTxnTooLarge: mysql.ErrTxnTooLarge, - mysql.ErrTxnRetryable: mysql.ErrTxnRetryable, - mysql.ErrWriteConflict: mysql.ErrWriteConflict, - mysql.ErrWriteConflictInTiDB: mysql.ErrWriteConflictInTiDB, - mysql.ErrCannotSetNilValue: mysql.ErrCannotSetNilValue, - mysql.ErrInvalidTxn: mysql.ErrInvalidTxn, - mysql.ErrEntryTooLarge: mysql.ErrEntryTooLarge, - mysql.ErrNotImplemented: mysql.ErrNotImplemented, - } - terror.ErrClassToMySQLCodes[terror.ClassKV] = kvMySQLErrCodes -} - // IsTxnRetryableError checks if the error could safely retry the transaction. func IsTxnRetryableError(err error) bool { if err == nil { diff --git a/meta/autoid/errors.go b/meta/autoid/errors.go index 40bec627d49e9..4ccf1cd952fef 100644 --- a/meta/autoid/errors.go +++ b/meta/autoid/errors.go @@ -28,19 +28,6 @@ var ( ErrAutoRandReadFailed = terror.ClassAutoid.New(mysql.ErrAutoRandReadFailed, mysql.MySQLErrName[mysql.ErrAutoRandReadFailed]) ) -func init() { - // Map error codes to mysql error codes. - tableMySQLErrCodes := map[terror.ErrCode]uint16{ - mysql.ErrAutoincReadFailed: mysql.ErrAutoincReadFailed, - mysql.ErrWrongAutoKey: mysql.ErrWrongAutoKey, - mysql.ErrInvalidTableID: mysql.ErrInvalidTableID, - mysql.ErrUnknownAllocatorType: mysql.ErrUnknownAllocatorType, - mysql.ErrAutoRandReadFailed: mysql.ErrAutoRandReadFailed, - mysql.ErrInvalidIncrementAndOffset: mysql.ErrInvalidIncrementAndOffset, - } - terror.ErrClassToMySQLCodes[terror.ClassAutoid] = tableMySQLErrCodes -} - const ( // AutoRandomPKisNotHandleErrMsg indicates the auto_random column attribute is defined on a non-primary key column, or the table's primary key is not a single integer column. AutoRandomPKisNotHandleErrMsg = "column %s is not the single integer primary key, or alter-primary-key is enabled" diff --git a/meta/meta.go b/meta/meta.go index b0c78c1bd8ee0..36b2010490e9d 100644 --- a/meta/meta.go +++ b/meta/meta.go @@ -900,13 +900,3 @@ func (m *Meta) SetSchemaDiff(diff *model.SchemaDiff) error { metrics.MetaHistogram.WithLabelValues(metrics.SetSchemaDiff, metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds()) return errors.Trace(err) } - -func init() { - metaMySQLErrCodes := map[terror.ErrCode]uint16{ - mysql.ErrDBCreateExists: mysql.ErrDBCreateExists, - mysql.ErrBadDB: mysql.ErrBadDB, - mysql.ErrNoSuchTable: mysql.ErrNoSuchTable, - mysql.ErrTableExists: mysql.ErrTableExists, - } - terror.ErrClassToMySQLCodes[terror.ClassMeta] = metaMySQLErrCodes -} diff --git a/planner/core/errors.go b/planner/core/errors.go index 8097e50285a0e..93b84f05e0ee6 100644 --- a/planner/core/errors.go +++ b/planner/core/errors.go @@ -82,69 +82,3 @@ var ( // Since we cannot know if user loggined with a password, use message of ErrAccessDeniedNoPassword instead ErrAccessDenied = terror.ClassOptimizer.New(mysql.ErrAccessDenied, mysql.MySQLErrName[mysql.ErrAccessDeniedNoPassword]) ) - -func init() { - mysqlErrCodeMap := map[terror.ErrCode]uint16{ - mysql.ErrViewInvalid: mysql.ErrViewInvalid, - mysql.ErrUnknown: mysql.ErrUnknown, - mysql.ErrTablenameNotAllowedHere: mysql.ErrTablenameNotAllowedHere, - mysql.ErrUnsupportedType: mysql.ErrUnsupportedType, - mysql.ErrAnalyzeMissIndex: mysql.ErrAnalyzeMissIndex, - mysql.ErrWrongParamCount: mysql.ErrWrongParamCount, - mysql.ErrSchemaChanged: mysql.ErrSchemaChanged, - mysql.ErrNotSupportedYet: mysql.ErrNotSupportedYet, - mysql.ErrWrongUsage: mysql.ErrWrongUsage, - mysql.ErrUnknownTable: mysql.ErrUnknownTable, - mysql.ErrWrongArguments: mysql.ErrWrongArguments, - mysql.ErrBadGeneratedColumn: mysql.ErrBadGeneratedColumn, - mysql.ErrFieldNotInGroupBy: mysql.ErrFieldNotInGroupBy, - mysql.ErrBadTable: mysql.ErrBadTable, - mysql.ErrKeyDoesNotExist: mysql.ErrKeyDoesNotExist, - mysql.ErrOperandColumns: mysql.ErrOperandColumns, - mysql.ErrInvalidGroupFuncUse: mysql.ErrInvalidGroupFuncUse, - mysql.ErrIllegalReference: mysql.ErrIllegalReference, - mysql.ErrNoDB: mysql.ErrNoDB, - mysql.ErrUnknownExplainFormat: mysql.ErrUnknownExplainFormat, - mysql.ErrWrongGroupField: mysql.ErrWrongGroupField, - mysql.ErrDupFieldName: mysql.ErrDupFieldName, - mysql.ErrNonUpdatableTable: mysql.ErrNonUpdatableTable, - mysql.ErrInternal: mysql.ErrInternal, - mysql.ErrMixOfGroupFuncAndFieldsIncompatible: mysql.ErrMixOfGroupFuncAndFieldsIncompatible, - mysql.ErrWrongNumberOfColumnsInSelect: mysql.ErrWrongNumberOfColumnsInSelect, - mysql.ErrWrongValueCountOnRow: mysql.ErrWrongValueCountOnRow, - mysql.ErrWindowInvalidWindowFuncUse: mysql.ErrWindowInvalidWindowFuncUse, - mysql.ErrWindowInvalidWindowFuncAliasUse: mysql.ErrWindowInvalidWindowFuncAliasUse, - mysql.ErrWindowNoSuchWindow: mysql.ErrWindowNoSuchWindow, - mysql.ErrWindowCircularityInWindowGraph: mysql.ErrWindowCircularityInWindowGraph, - mysql.ErrWindowNoChildPartitioning: mysql.ErrWindowNoChildPartitioning, - mysql.ErrWindowNoInherentFrame: mysql.ErrWindowNoInherentFrame, - mysql.ErrWindowNoRedefineOrderBy: mysql.ErrWindowNoRedefineOrderBy, - mysql.ErrWindowDuplicateName: mysql.ErrWindowDuplicateName, - mysql.ErrPartitionClauseOnNonpartitioned: mysql.ErrPartitionClauseOnNonpartitioned, - mysql.ErrDBaccessDenied: mysql.ErrDBaccessDenied, - mysql.ErrTableaccessDenied: mysql.ErrTableaccessDenied, - mysql.ErrSpecificAccessDenied: mysql.ErrSpecificAccessDenied, - mysql.ErrViewNoExplain: mysql.ErrViewNoExplain, - mysql.ErrWindowFrameStartIllegal: mysql.ErrWindowFrameStartIllegal, - mysql.ErrWindowFrameEndIllegal: mysql.ErrWindowFrameEndIllegal, - mysql.ErrWindowFrameIllegal: mysql.ErrWindowFrameIllegal, - mysql.ErrWindowRangeFrameOrderType: mysql.ErrWindowRangeFrameOrderType, - mysql.ErrWindowRangeFrameTemporalType: mysql.ErrWindowRangeFrameTemporalType, - mysql.ErrWindowRangeFrameNumericType: mysql.ErrWindowRangeFrameNumericType, - mysql.ErrWindowRangeBoundNotConstant: mysql.ErrWindowRangeBoundNotConstant, - mysql.ErrWindowRowsIntervalUse: mysql.ErrWindowRowsIntervalUse, - mysql.ErrWindowFunctionIgnoresFrame: mysql.ErrWindowFunctionIgnoresFrame, - mysql.ErrUnsupportedOnGeneratedColumn: mysql.ErrUnsupportedOnGeneratedColumn, - mysql.ErrNoSuchThread: mysql.ErrNoSuchThread, - mysql.ErrAccessDenied: mysql.ErrAccessDenied, - mysql.ErrPrivilegeCheckFail: mysql.ErrPrivilegeCheckFail, - mysql.ErrCartesianProductUnsupported: mysql.ErrCartesianProductUnsupported, - mysql.ErrPreparedStmtNotFound: mysql.ErrPreparedStmtNotFound, - mysql.ErrNonUniq: mysql.ErrNonUniq, - mysql.ErrBadField: mysql.ErrBadField, - mysql.ErrNonuniqTable: mysql.ErrNonuniqTable, - mysql.ErrTooBigPrecision: mysql.ErrTooBigPrecision, - mysql.ErrInvalidWildCard: mysql.ErrInvalidWildCard, - } - terror.ErrClassToMySQLCodes[terror.ClassOptimizer] = mysqlErrCodeMap -} diff --git a/plugin/errors.go b/plugin/errors.go index af6062cd26a88..1fa05a13165ab 100644 --- a/plugin/errors.go +++ b/plugin/errors.go @@ -19,30 +19,11 @@ import ( ) var ( - errInvalidPluginID = createPluginError(mysql.ErrInvalidPluginID) - errInvalidPluginManifest = createPluginError(mysql.ErrInvalidPluginManifest) - errInvalidPluginName = createPluginError(mysql.ErrInvalidPluginName) - errInvalidPluginVersion = createPluginError(mysql.ErrInvalidPluginVersion) - errDuplicatePlugin = createPluginError(mysql.ErrDuplicatePlugin) - errInvalidPluginSysVarName = createPluginError(mysql.ErrInvalidPluginSysVarName) - errRequireVersionCheckFail = createPluginError(mysql.ErrRequireVersionCheckFail) + errInvalidPluginID = terror.ClassPlugin.NewStd(mysql.ErrInvalidPluginID) + errInvalidPluginManifest = terror.ClassPlugin.NewStd(mysql.ErrInvalidPluginManifest) + errInvalidPluginName = terror.ClassPlugin.NewStd(mysql.ErrInvalidPluginName) + errInvalidPluginVersion = terror.ClassPlugin.NewStd(mysql.ErrInvalidPluginVersion) + errDuplicatePlugin = terror.ClassPlugin.NewStd(mysql.ErrDuplicatePlugin) + errInvalidPluginSysVarName = terror.ClassPlugin.NewStd(mysql.ErrInvalidPluginSysVarName) + errRequireVersionCheckFail = terror.ClassPlugin.NewStd(mysql.ErrRequireVersionCheckFail) ) - -func createPluginError(code terror.ErrCode) *terror.Error { - return terror.ClassPlugin.New(code, mysql.MySQLErrName[uint16(code)]) -} - -func init() { - pluginMySQLErrCodes := map[terror.ErrCode]uint16{ - mysql.ErrInvalidPluginID: mysql.ErrInvalidPluginID, - mysql.ErrInvalidPluginManifest: mysql.ErrInvalidPluginManifest, - mysql.ErrInvalidPluginName: mysql.ErrInvalidPluginName, - mysql.ErrInvalidPluginVersion: mysql.ErrInvalidPluginVersion, - mysql.ErrDuplicatePlugin: mysql.ErrDuplicatePlugin, - mysql.ErrInvalidPluginSysVarName: mysql.ErrInvalidPluginSysVarName, - mysql.ErrRequireVersionCheckFail: mysql.ErrRequireVersionCheckFail, - mysql.ErrUnsupportedReloadPlugin: mysql.ErrUnsupportedReloadPlugin, - mysql.ErrUnsupportedReloadPluginVar: mysql.ErrUnsupportedReloadPluginVar, - } - terror.ErrClassToMySQLCodes[terror.ClassPlugin] = pluginMySQLErrCodes -} diff --git a/privilege/privileges/errors.go b/privilege/privileges/errors.go index 63eea01f2b70c..7303499905d1b 100644 --- a/privilege/privileges/errors.go +++ b/privilege/privileges/errors.go @@ -18,17 +18,9 @@ import ( "github.com/pingcap/parser/terror" ) +// error definitions. var ( errInvalidPrivilegeType = terror.ClassPrivilege.New(mysql.ErrInvalidPrivilegeType, mysql.MySQLErrName[mysql.ErrInvalidPrivilegeType]) - errNonexistingGrant = terror.ClassPrivilege.New(mysql.ErrNonexistingGrant, mysql.MySQLErrName[mysql.ErrNonexistingGrant]) + ErrNonexistingGrant = terror.ClassPrivilege.New(mysql.ErrNonexistingGrant, mysql.MySQLErrName[mysql.ErrNonexistingGrant]) errLoadPrivilege = terror.ClassPrivilege.New(mysql.ErrLoadPrivilege, mysql.MySQLErrName[mysql.ErrLoadPrivilege]) ) - -func init() { - privilegeMySQLErrCodes := map[terror.ErrCode]uint16{ - mysql.ErrNonexistingGrant: mysql.ErrNonexistingGrant, - mysql.ErrLoadPrivilege: mysql.ErrLoadPrivilege, - mysql.ErrInvalidPrivilegeType: mysql.ErrInvalidPrivilegeType, - } - terror.ErrClassToMySQLCodes[terror.ClassPrivilege] = privilegeMySQLErrCodes -} diff --git a/privilege/privileges/privileges.go b/privilege/privileges/privileges.go index 144e686548ebb..507ef60d5c103 100644 --- a/privilege/privileges/privileges.go +++ b/privilege/privileges/privileges.go @@ -299,7 +299,7 @@ func (p *UserPrivileges) UserPrivilegesTable() [][]types.Datum { // ShowGrants implements privilege.Manager ShowGrants interface. func (p *UserPrivileges) ShowGrants(ctx sessionctx.Context, user *auth.UserIdentity, roles []*auth.RoleIdentity) (grants []string, err error) { if SkipWithGrant { - return nil, errNonexistingGrant.GenWithStackByArgs("root", "%") + return nil, ErrNonexistingGrant.GenWithStackByArgs("root", "%") } mysqlPrivilege := p.Handle.Get() u := user.Username @@ -310,7 +310,7 @@ func (p *UserPrivileges) ShowGrants(ctx sessionctx.Context, user *auth.UserIdent } grants = mysqlPrivilege.showGrants(u, h, roles) if len(grants) == 0 { - err = errNonexistingGrant.GenWithStackByArgs(u, h) + err = ErrNonexistingGrant.GenWithStackByArgs(u, h) } return diff --git a/privilege/privileges/privileges_test.go b/privilege/privileges/privileges_test.go index 07767c3cb0d81..65dfc7cec0a75 100644 --- a/privilege/privileges/privileges_test.go +++ b/privilege/privileges/privileges_test.go @@ -31,6 +31,7 @@ import ( "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/privilege" + "github.com/pingcap/tidb/privilege/privileges" "github.com/pingcap/tidb/session" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/store/mockstore" @@ -338,8 +339,7 @@ func (s *testPrivilegeSuite) TestShowGrants(c *C) { _, err = pc.ShowGrants(se, &auth.UserIdentity{Username: "show", Hostname: "localhost"}, nil) c.Assert(err, NotNil) // cant show grants for non-existent - errNonexistingGrant := terror.ClassPrivilege.New(mysql.ErrNonexistingGrant, mysql.MySQLErrName[mysql.ErrNonexistingGrant]) - c.Assert(terror.ErrorEqual(err, errNonexistingGrant), IsTrue) + c.Assert(terror.ErrorEqual(err, privileges.ErrNonexistingGrant), IsTrue) // Test SHOW GRANTS with USING roles. mustExec(c, se, `CREATE ROLE 'r1', 'r2'`) diff --git a/server/server.go b/server/server.go index 879a995ffdcdc..4eb1a359b4330 100644 --- a/server/server.go +++ b/server/server.go @@ -672,15 +672,3 @@ const ( codeInvalidSequence = 3 codeInvalidType = 4 ) - -func init() { - serverMySQLErrCodes := map[terror.ErrCode]uint16{ - mysql.ErrNotAllowedCommand: mysql.ErrNotAllowedCommand, - mysql.ErrAccessDenied: mysql.ErrAccessDenied, - mysql.ErrUnknownFieldType: mysql.ErrUnknownFieldType, - mysql.ErrInvalidSequence: mysql.ErrInvalidSequence, - mysql.ErrInvalidType: mysql.ErrInvalidType, - mysql.ErrConCount: mysql.ErrConCount, - } - terror.ErrClassToMySQLCodes[terror.ClassServer] = serverMySQLErrCodes -} diff --git a/session/tidb.go b/session/tidb.go index 2b5f475158a34..8aed73626af10 100644 --- a/session/tidb.go +++ b/session/tidb.go @@ -379,10 +379,3 @@ func ResultSetToStringSlice(ctx context.Context, s Session, rs sqlexec.RecordSet var ( ErrForUpdateCantRetry = terror.ClassSession.New(mysql.ErrForUpdateCantRetry, mysql.MySQLErrName[mysql.ErrForUpdateCantRetry]) ) - -func init() { - sessionMySQLErrCodes := map[terror.ErrCode]uint16{ - mysql.ErrForUpdateCantRetry: mysql.ErrForUpdateCantRetry, - } - terror.ErrClassToMySQLCodes[terror.ClassSession] = sessionMySQLErrCodes -} diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go index 74f9b3e0e0343..e342861f7880c 100644 --- a/sessionctx/variable/sysvar.go +++ b/sessionctx/variable/sysvar.go @@ -81,24 +81,6 @@ func init() { SysVars[v.Name] = v } initSynonymsSysVariables() - - // Register terror to mysql error map. - mySQLErrCodes := map[terror.ErrCode]uint16{ - mysql.ErrCantGetValidID: mysql.ErrCantGetValidID, - mysql.ErrCantSetToNull: mysql.ErrCantSetToNull, - mysql.ErrSnapshotTooOld: mysql.ErrSnapshotTooOld, - mysql.ErrUnsupportedValueForVar: mysql.ErrUnsupportedValueForVar, - mysql.ErrUnknownSystemVariable: mysql.ErrUnknownSystemVariable, - mysql.ErrIncorrectGlobalLocalVar: mysql.ErrIncorrectGlobalLocalVar, - mysql.ErrUnknownTimeZone: mysql.ErrUnknownTimeZone, - mysql.ErrVariableIsReadonly: mysql.ErrVariableIsReadonly, - mysql.ErrWrongValueForVar: mysql.ErrWrongValueForVar, - mysql.ErrWrongTypeForVar: mysql.ErrWrongTypeForVar, - mysql.ErrTruncatedWrongValue: mysql.ErrTruncatedWrongValue, - mysql.ErrMaxPreparedStmtCountReached: mysql.ErrMaxPreparedStmtCountReached, - mysql.ErrUnsupportedIsolationLevel: mysql.ErrUnsupportedIsolationLevel, - } - terror.ErrClassToMySQLCodes[terror.ClassVariable] = mySQLErrCodes } // BoolToIntStr converts bool to int string, for example "0" or "1". diff --git a/store/tikv/backoff.go b/store/tikv/backoff.go index 6b121fee24174..830588e2dd10f 100644 --- a/store/tikv/backoff.go +++ b/store/tikv/backoff.go @@ -24,8 +24,6 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/log" - "github.com/pingcap/parser/mysql" - "github.com/pingcap/parser/terror" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/util/logutil" @@ -199,7 +197,7 @@ func (t backoffType) TError() error { case boServerBusy: return ErrTiKVServerBusy } - return terror.ClassTiKV.New(mysql.ErrUnknown, mysql.MySQLErrName[mysql.ErrUnknown]) + return ErrUnknown } // Maximum total sleep time(in ms) for kv/cop commands. diff --git a/store/tikv/error.go b/store/tikv/error.go index 790bc01d54bbb..b4f1900f4277c 100644 --- a/store/tikv/error.go +++ b/store/tikv/error.go @@ -42,6 +42,14 @@ var ( ErrLockAcquireFailAndNoWaitSet = terror.ClassTiKV.New(mysql.ErrLockAcquireFailAndNoWaitSet, mysql.MySQLErrName[mysql.ErrLockAcquireFailAndNoWaitSet]) ErrLockWaitTimeout = terror.ClassTiKV.New(mysql.ErrLockWaitTimeout, mysql.MySQLErrName[mysql.ErrLockWaitTimeout]) ErrTokenLimit = terror.ClassTiKV.New(mysql.ErrTiKVStoreLimit, mysql.MySQLErrName[mysql.ErrTiKVStoreLimit]) + ErrUnknown = terror.ClassTiKV.New(mysql.ErrUnknown, mysql.MySQLErrName[mysql.ErrUnknown]) +) + +// Registers error returned from TiKV. +var ( + _ = terror.ClassTiKV.NewStd(mysql.ErrDataOutOfRange) + _ = terror.ClassTiKV.NewStd(mysql.ErrTruncatedWrongValue) + _ = terror.ClassTiKV.NewStd(mysql.ErrDivisionByZero) ) // ErrDeadlock wraps *kvrpcpb.Deadlock to implement the error interface. @@ -54,20 +62,3 @@ type ErrDeadlock struct { func (d *ErrDeadlock) Error() string { return d.Deadlock.String() } - -func init() { - tikvMySQLErrCodes := map[terror.ErrCode]uint16{ - mysql.ErrTiKVServerTimeout: mysql.ErrTiKVServerTimeout, - mysql.ErrResolveLockTimeout: mysql.ErrResolveLockTimeout, - mysql.ErrPDServerTimeout: mysql.ErrPDServerTimeout, - mysql.ErrRegionUnavailable: mysql.ErrRegionUnavailable, - mysql.ErrTiKVServerBusy: mysql.ErrTiKVServerBusy, - mysql.ErrGCTooEarly: mysql.ErrGCTooEarly, - mysql.ErrTruncatedWrongValue: mysql.ErrTruncatedWrongValue, - mysql.ErrQueryInterrupted: mysql.ErrQueryInterrupted, - mysql.ErrLockAcquireFailAndNoWaitSet: mysql.ErrLockAcquireFailAndNoWaitSet, - mysql.ErrDataOutOfRange: mysql.ErrDataOutOfRange, - mysql.ErrLockWaitTimeout: mysql.ErrLockWaitTimeout, - } - terror.ErrClassToMySQLCodes[terror.ClassTiKV] = tikvMySQLErrCodes -} diff --git a/structure/structure.go b/structure/structure.go index 7617e08eb7813..0191ce0a776e6 100644 --- a/structure/structure.go +++ b/structure/structure.go @@ -46,14 +46,3 @@ type TxStructure struct { readWriter kv.RetrieverMutator prefix []byte } - -func init() { - // Register terror to mysql error map. - mySQLErrCodes := map[terror.ErrCode]uint16{ - mysql.ErrInvalidHashKeyFlag: mysql.ErrInvalidHashKeyFlag, - mysql.ErrInvalidListIndex: mysql.ErrInvalidListIndex, - mysql.ErrInvalidListMetaData: mysql.ErrInvalidListMetaData, - mysql.ErrWriteOnSnapshot: mysql.ErrWriteOnSnapshot, - } - terror.ErrClassToMySQLCodes[terror.ClassStructure] = mySQLErrCodes -} diff --git a/table/table.go b/table/table.go index a6434ed134b59..98f682498abe6 100644 --- a/table/table.go +++ b/table/table.go @@ -278,26 +278,3 @@ func (s Slice) Less(i, j int) bool { } func (s Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - -func init() { - tableMySQLErrCodes := map[terror.ErrCode]uint16{ - mysql.ErrBadNull: mysql.ErrBadNull, - mysql.ErrBadField: mysql.ErrBadField, - mysql.ErrFieldSpecifiedTwice: mysql.ErrFieldSpecifiedTwice, - mysql.ErrNoDefaultForField: mysql.ErrNoDefaultForField, - mysql.ErrTruncatedWrongValueForField: mysql.ErrTruncatedWrongValueForField, - mysql.ErrUnknownPartition: mysql.ErrUnknownPartition, - mysql.ErrNoPartitionForGivenValue: mysql.ErrNoPartitionForGivenValue, - mysql.ErrLockOrActiveTransaction: mysql.ErrLockOrActiveTransaction, - mysql.ErrIndexOutBound: mysql.ErrIndexOutBound, - mysql.ErrColumnStateNonPublic: mysql.ErrColumnStateNonPublic, - mysql.ErrFieldGetDefaultFailed: mysql.ErrFieldGetDefaultFailed, - mysql.ErrUnsupportedOp: mysql.ErrUnsupportedOp, - mysql.ErrRowNotFound: mysql.ErrRowNotFound, - mysql.ErrTableStateCantNone: mysql.ErrTableStateCantNone, - mysql.ErrColumnStateCantNone: mysql.ErrColumnStateCantNone, - mysql.ErrIndexStateCantNone: mysql.ErrIndexStateCantNone, - mysql.ErrInvalidRecordKey: mysql.ErrInvalidRecordKey, - } - terror.ErrClassToMySQLCodes[terror.ClassTable] = tableMySQLErrCodes -} diff --git a/tablecodec/tablecodec.go b/tablecodec/tablecodec.go index 671a5a624dbdd..2cbc894d88717 100644 --- a/tablecodec/tablecodec.go +++ b/tablecodec/tablecodec.go @@ -767,12 +767,3 @@ func GetTableIndexKeyRange(tableID, indexID int64) (startKey, endKey []byte) { endKey = EncodeIndexSeekKey(tableID, indexID, []byte{255}) return } - -func init() { - mySQLErrCodes := map[terror.ErrCode]uint16{ - mysql.ErrInvalidKey: mysql.ErrInvalidKey, - mysql.ErrInvalidRecordKey: mysql.ErrInvalidRecordKey, - mysql.ErrInvalidIndexKey: mysql.ErrInvalidIndexKey, - } - terror.ErrClassToMySQLCodes[terror.ClassXEval] = mySQLErrCodes -} diff --git a/types/errors.go b/types/errors.go index 21e202e82cfd1..92859f995bca7 100644 --- a/types/errors.go +++ b/types/errors.go @@ -76,34 +76,3 @@ var ( // ErrWrongValue is returned when the input value is in wrong format. ErrWrongValue = terror.ClassTypes.New(mysql.ErrWrongValue, mysql.MySQLErrName[mysql.ErrWrongValue]) ) - -func init() { - typesMySQLErrCodes := map[terror.ErrCode]uint16{ - mysql.ErrInvalidDefault: mysql.ErrInvalidDefault, - mysql.ErrDataTooLong: mysql.ErrDataTooLong, - mysql.ErrIllegalValueForType: mysql.ErrIllegalValueForType, - mysql.WarnDataTruncated: mysql.WarnDataTruncated, - mysql.ErrDataOutOfRange: mysql.ErrDataOutOfRange, - mysql.ErrDivisionByZero: mysql.ErrDivisionByZero, - mysql.ErrTooBigDisplaywidth: mysql.ErrTooBigDisplaywidth, - mysql.ErrTooBigFieldlength: mysql.ErrTooBigFieldlength, - mysql.ErrTooBigSet: mysql.ErrTooBigSet, - mysql.ErrTooBigScale: mysql.ErrTooBigScale, - mysql.ErrTooBigPrecision: mysql.ErrTooBigPrecision, - mysql.ErrBadNumber: mysql.ErrBadNumber, - mysql.ErrInvalidFieldSize: mysql.ErrInvalidFieldSize, - mysql.ErrMBiggerThanD: mysql.ErrMBiggerThanD, - mysql.ErrWarnDataOutOfRange: mysql.ErrWarnDataOutOfRange, - mysql.ErrDuplicatedValueInType: mysql.ErrDuplicatedValueInType, - mysql.ErrDatetimeFunctionOverflow: mysql.ErrDatetimeFunctionOverflow, - mysql.ErrCastAsSignedOverflow: mysql.ErrCastAsSignedOverflow, - mysql.ErrCastNegIntAsUnsigned: mysql.ErrCastNegIntAsUnsigned, - mysql.ErrInvalidYearFormat: mysql.ErrInvalidYearFormat, - mysql.ErrInvalidYear: mysql.ErrInvalidYear, - mysql.ErrTruncatedWrongValue: mysql.ErrTruncatedWrongValue, - mysql.ErrInvalidTimeFormat: mysql.ErrInvalidTimeFormat, - mysql.ErrInvalidWeekModeFormat: mysql.ErrInvalidWeekModeFormat, - mysql.ErrWrongValue: mysql.ErrWrongValue, - } - terror.ErrClassToMySQLCodes[terror.ClassTypes] = typesMySQLErrCodes -} diff --git a/types/json/constants.go b/types/json/constants.go index 32fb770f3f946..3e276c918b5b4 100644 --- a/types/json/constants.go +++ b/types/json/constants.go @@ -223,19 +223,6 @@ var ( ErrUnsupportedSecondArgumentType = terror.ClassJSON.New(mysql.ErrUnsupportedSecondArgumentType, mysql.MySQLErrName[mysql.ErrUnsupportedSecondArgumentType]) ) -func init() { - terror.ErrClassToMySQLCodes[terror.ClassJSON] = map[terror.ErrCode]uint16{ - mysql.ErrInvalidJSONText: mysql.ErrInvalidJSONText, - mysql.ErrInvalidJSONPath: mysql.ErrInvalidJSONPath, - mysql.ErrInvalidJSONData: mysql.ErrInvalidJSONData, - mysql.ErrInvalidJSONPathWildcard: mysql.ErrInvalidJSONPathWildcard, - mysql.ErrInvalidJSONContainsPathType: mysql.ErrInvalidJSONContainsPathType, - mysql.ErrJSONDocumentNULLKey: mysql.ErrJSONDocumentNULLKey, - mysql.ErrInvalidJSONPathArrayCell: mysql.ErrInvalidJSONPathArrayCell, - mysql.ErrUnsupportedSecondArgumentType: mysql.ErrUnsupportedSecondArgumentType, - } -} - // json_contains_path function type choices // See: https://dev.mysql.com/doc/refman/5.7/en/json-search-functions.html#function_json-contains-path const ( diff --git a/util/admin/admin.go b/util/admin/admin.go index 304c2031d6869..1155f4e3930c0 100644 --- a/util/admin/admin.go +++ b/util/admin/admin.go @@ -447,14 +447,3 @@ var ( // ErrCannotCancelDDLJob returns when cancel a almost finished ddl job, because cancel in now may cause data inconsistency. ErrCannotCancelDDLJob = terror.ClassAdmin.New(mysql.ErrCannotCancelDDLJob, mysql.MySQLErrName[mysql.ErrCannotCancelDDLJob]) ) - -func init() { - // Register terror to mysql error map. - mySQLErrCodes := map[terror.ErrCode]uint16{ - mysql.ErrDataInConsistent: mysql.ErrDataInConsistent, - mysql.ErrDDLJobNotFound: mysql.ErrDDLJobNotFound, - mysql.ErrCancelFinishedDDLJob: mysql.ErrCancelFinishedDDLJob, - mysql.ErrCannotCancelDDLJob: mysql.ErrCannotCancelDDLJob, - } - terror.ErrClassToMySQLCodes[terror.ClassAdmin] = mySQLErrCodes -} diff --git a/util/memory/action.go b/util/memory/action.go index 6c64a2667fdf0..5fe62f0ea97ec 100644 --- a/util/memory/action.go +++ b/util/memory/action.go @@ -107,10 +107,3 @@ const ( // PanicMemoryExceed represents the panic message when out of memory quota. PanicMemoryExceed string = "Out Of Memory Quota!" ) - -func init() { - errCodes := map[terror.ErrCode]uint16{ - mysql.ErrMemExceedThreshold: mysql.ErrMemExceedThreshold, - } - terror.ErrClassToMySQLCodes[terror.ClassUtil] = errCodes -} From d9cf0b6cab01a4e614a3fae95448df1cf4ab3b2f Mon Sep 17 00:00:00 2001 From: qupeng Date: Thu, 6 Feb 2020 17:31:53 +0800 Subject: [PATCH 2/5] tikvclient: fix a bug about unknown response from tikv (#14650) --- store/tikv/client_batch.go | 2 +- store/tikv/tikvrpc/tikvrpc.go | 57 ++++++++++++++++-------------- store/tikv/tikvrpc/tikvrpc_test.go | 31 ++++++++++++++++ 3 files changed, 62 insertions(+), 28 deletions(-) create mode 100644 store/tikv/tikvrpc/tikvrpc_test.go diff --git a/store/tikv/client_batch.go b/store/tikv/client_batch.go index f625103f919e2..0626be0e690ed 100644 --- a/store/tikv/client_batch.go +++ b/store/tikv/client_batch.go @@ -588,7 +588,7 @@ func sendBatchRequest( if !ok { return nil, errors.Trace(entry.err) } - return tikvrpc.FromBatchCommandsResponse(res), nil + return tikvrpc.FromBatchCommandsResponse(res) case <-ctx1.Done(): atomic.StoreInt32(&entry.canceled, 1) logutil.BgLogger().Warn("wait response is cancelled", diff --git a/store/tikv/tikvrpc/tikvrpc.go b/store/tikv/tikvrpc/tikvrpc.go index 064dd9224d515..7dbd0d98d5fb0 100644 --- a/store/tikv/tikvrpc/tikvrpc.go +++ b/store/tikv/tikvrpc/tikvrpc.go @@ -426,60 +426,63 @@ type Response struct { } // FromBatchCommandsResponse converts a BatchCommands response to Response. -func FromBatchCommandsResponse(res *tikvpb.BatchCommandsResponse_Response) *Response { +func FromBatchCommandsResponse(res *tikvpb.BatchCommandsResponse_Response) (*Response, error) { + if res.GetCmd() == nil { + return nil, errors.New("Unknown command response") + } switch res := res.GetCmd().(type) { case *tikvpb.BatchCommandsResponse_Response_Get: - return &Response{Resp: res.Get} + return &Response{Resp: res.Get}, nil case *tikvpb.BatchCommandsResponse_Response_Scan: - return &Response{Resp: res.Scan} + return &Response{Resp: res.Scan}, nil case *tikvpb.BatchCommandsResponse_Response_Prewrite: - return &Response{Resp: res.Prewrite} + return &Response{Resp: res.Prewrite}, nil case *tikvpb.BatchCommandsResponse_Response_Commit: - return &Response{Resp: res.Commit} + return &Response{Resp: res.Commit}, nil case *tikvpb.BatchCommandsResponse_Response_Cleanup: - return &Response{Resp: res.Cleanup} + return &Response{Resp: res.Cleanup}, nil case *tikvpb.BatchCommandsResponse_Response_BatchGet: - return &Response{Resp: res.BatchGet} + return &Response{Resp: res.BatchGet}, nil case *tikvpb.BatchCommandsResponse_Response_BatchRollback: - return &Response{Resp: res.BatchRollback} + return &Response{Resp: res.BatchRollback}, nil case *tikvpb.BatchCommandsResponse_Response_ScanLock: - return &Response{Resp: res.ScanLock} + return &Response{Resp: res.ScanLock}, nil case *tikvpb.BatchCommandsResponse_Response_ResolveLock: - return &Response{Resp: res.ResolveLock} + return &Response{Resp: res.ResolveLock}, nil case *tikvpb.BatchCommandsResponse_Response_GC: - return &Response{Resp: res.GC} + return &Response{Resp: res.GC}, nil case *tikvpb.BatchCommandsResponse_Response_DeleteRange: - return &Response{Resp: res.DeleteRange} + return &Response{Resp: res.DeleteRange}, nil case *tikvpb.BatchCommandsResponse_Response_RawGet: - return &Response{Resp: res.RawGet} + return &Response{Resp: res.RawGet}, nil case *tikvpb.BatchCommandsResponse_Response_RawBatchGet: - return &Response{Resp: res.RawBatchGet} + return &Response{Resp: res.RawBatchGet}, nil case *tikvpb.BatchCommandsResponse_Response_RawPut: - return &Response{Resp: res.RawPut} + return &Response{Resp: res.RawPut}, nil case *tikvpb.BatchCommandsResponse_Response_RawBatchPut: - return &Response{Resp: res.RawBatchPut} + return &Response{Resp: res.RawBatchPut}, nil case *tikvpb.BatchCommandsResponse_Response_RawDelete: - return &Response{Resp: res.RawDelete} + return &Response{Resp: res.RawDelete}, nil case *tikvpb.BatchCommandsResponse_Response_RawBatchDelete: - return &Response{Resp: res.RawBatchDelete} + return &Response{Resp: res.RawBatchDelete}, nil case *tikvpb.BatchCommandsResponse_Response_RawDeleteRange: - return &Response{Resp: res.RawDeleteRange} + return &Response{Resp: res.RawDeleteRange}, nil case *tikvpb.BatchCommandsResponse_Response_RawScan: - return &Response{Resp: res.RawScan} + return &Response{Resp: res.RawScan}, nil case *tikvpb.BatchCommandsResponse_Response_Coprocessor: - return &Response{Resp: res.Coprocessor} + return &Response{Resp: res.Coprocessor}, nil case *tikvpb.BatchCommandsResponse_Response_PessimisticLock: - return &Response{Resp: res.PessimisticLock} + return &Response{Resp: res.PessimisticLock}, nil case *tikvpb.BatchCommandsResponse_Response_PessimisticRollback: - return &Response{Resp: res.PessimisticRollback} + return &Response{Resp: res.PessimisticRollback}, nil case *tikvpb.BatchCommandsResponse_Response_Empty: - return &Response{Resp: res.Empty} + return &Response{Resp: res.Empty}, nil case *tikvpb.BatchCommandsResponse_Response_TxnHeartBeat: - return &Response{Resp: res.TxnHeartBeat} + return &Response{Resp: res.TxnHeartBeat}, nil case *tikvpb.BatchCommandsResponse_Response_CheckTxnStatus: - return &Response{Resp: res.CheckTxnStatus} + return &Response{Resp: res.CheckTxnStatus}, nil } - return nil + panic("unreachable") } // CopStreamResponse combinates tikvpb.Tikv_CoprocessorStreamClient and the first Recv() result together. diff --git a/store/tikv/tikvrpc/tikvrpc_test.go b/store/tikv/tikvrpc/tikvrpc_test.go new file mode 100644 index 0000000000000..a9dbd84c579d8 --- /dev/null +++ b/store/tikv/tikvrpc/tikvrpc_test.go @@ -0,0 +1,31 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package tikvrpc + +import ( + "github.com/pingcap/kvproto/pkg/tikvpb" + + . "github.com/pingcap/check" +) + +type testBatchCommand struct{} + +var _ = Suite(&testBatchCommand{}) + +func (s *testBatchCommand) TestBatchResponse(c *C) { + resp := &tikvpb.BatchCommandsResponse_Response{} + batchResp, err := FromBatchCommandsResponse(resp) + c.Assert(batchResp == nil, IsTrue) + c.Assert(err != nil, IsTrue) +} From 3c59e7ce561bb395232310b57dad1b24ef3a39ba Mon Sep 17 00:00:00 2001 From: Yixuan Zhao Date: Fri, 7 Feb 2020 01:50:24 +0800 Subject: [PATCH 3/5] ddl:Revert "ddl: speed up ddl test" (#14658) --- ddl/db_partition_test.go | 2 +- ddl/db_test.go | 30 ++++++++++++------------------ 2 files changed, 13 insertions(+), 19 deletions(-) diff --git a/ddl/db_partition_test.go b/ddl/db_partition_test.go index b27b4ffc5fb25..9bec95d1dabe2 100644 --- a/ddl/db_partition_test.go +++ b/ddl/db_partition_test.go @@ -1405,7 +1405,7 @@ func (s *testIntegrationSuite5) TestPartitionAddPrimaryKey(c *C) { testPartitionAddIndexOrPK(c, tk, "primary key") } -func (s *testIntegrationSuite6) TestPartitionAddIndex(c *C) { +func (s *testIntegrationSuite1) TestPartitionAddIndex(c *C) { tk := testkit.NewTestKit(c, s.store) testPartitionAddIndexOrPK(c, tk, "index") } diff --git a/ddl/db_test.go b/ddl/db_test.go index 927e63488f096..1932007036f03 100644 --- a/ddl/db_test.go +++ b/ddl/db_test.go @@ -67,9 +67,6 @@ var _ = Suite(&testDBSuite3{&testDBSuite{}}) var _ = Suite(&testDBSuite4{&testDBSuite{}}) var _ = Suite(&testDBSuite5{&testDBSuite{}}) var _ = Suite(&testDBSuite6{&testDBSuite{}}) -var _ = Suite(&testDBSuite7{&testDBSuite{}}) -var _ = Suite(&testDBSuite8{&testDBSuite{}}) -var _ = Suite(&testDBSuite9{&testDBSuite{}}) const defaultBatchSize = 1024 @@ -137,9 +134,6 @@ type testDBSuite3 struct{ *testDBSuite } type testDBSuite4 struct{ *testDBSuite } type testDBSuite5 struct{ *testDBSuite } type testDBSuite6 struct{ *testDBSuite } -type testDBSuite7 struct{ *testDBSuite } -type testDBSuite8 struct{ *testDBSuite } -type testDBSuite9 struct{ *testDBSuite } func (s *testDBSuite4) TestAddIndexWithPK(c *C) { s.tk = testkit.NewTestKit(c, s.store) @@ -230,7 +224,7 @@ func backgroundExec(s kv.Storage, sql string, done chan error) { } // TestAddPrimaryKeyRollback1 is used to test scenarios that will roll back when a duplicate primary key is encountered. -func (s *testDBSuite7) TestAddPrimaryKeyRollback1(c *C) { +func (s *testDBSuite5) TestAddPrimaryKeyRollback1(c *C) { hasNullValsInKey := false idxName := "PRIMARY" addIdxSQL := "alter table t1 add primary key c3_index (c3);" @@ -239,7 +233,7 @@ func (s *testDBSuite7) TestAddPrimaryKeyRollback1(c *C) { } // TestAddPrimaryKeyRollback2 is used to test scenarios that will roll back when a null primary key is encountered. -func (s *testDBSuite9) TestAddPrimaryKeyRollback2(c *C) { +func (s *testDBSuite1) TestAddPrimaryKeyRollback2(c *C) { hasNullValsInKey := true idxName := "PRIMARY" addIdxSQL := "alter table t1 add primary key c3_index (c3);" @@ -359,7 +353,7 @@ func (s *testDBSuite5) TestCancelAddPrimaryKey(c *C) { tk.MustExec("drop table t1") } -func (s *testDBSuite9) TestCancelAddIndex(c *C) { +func (s *testDBSuite3) TestCancelAddIndex(c *C) { idxName := "c3_index " addIdxSQL := "create unique index c3_index on t1 (c3)" testCancelAddIndex(c, s.store, s.dom.DDL(), s.lease, idxName, addIdxSQL, "") @@ -895,12 +889,12 @@ func (s *testDBSuite5) TestAddMultiColumnsIndex(c *C) { s.tk.MustExec("alter table tidb.test add index idx1 (a, b);") s.tk.MustExec("admin check table test") } -func (s *testDBSuite6) TestAddPrimaryKey1(c *C) { +func (s *testDBSuite1) TestAddPrimaryKey1(c *C) { testAddIndex(c, s.store, s.lease, false, "create table test_add_index (c1 bigint, c2 bigint, c3 bigint, unique key(c1))", "primary") } -func (s *testDBSuite6) TestAddPrimaryKey2(c *C) { +func (s *testDBSuite2) TestAddPrimaryKey2(c *C) { testAddIndex(c, s.store, s.lease, true, `create table test_add_index (c1 bigint, c2 bigint, c3 bigint, key(c1)) partition by range (c3) ( @@ -911,13 +905,13 @@ func (s *testDBSuite6) TestAddPrimaryKey2(c *C) { partition p4 values less than maxvalue)`, "primary") } -func (s *testDBSuite9) TestAddPrimaryKey3(c *C) { +func (s *testDBSuite3) TestAddPrimaryKey3(c *C) { testAddIndex(c, s.store, s.lease, true, `create table test_add_index (c1 bigint, c2 bigint, c3 bigint, key(c1)) partition by hash (c3) partitions 4;`, "primary") } -func (s *testDBSuite7) TestAddPrimaryKey4(c *C) { +func (s *testDBSuite4) TestAddPrimaryKey4(c *C) { testAddIndex(c, s.store, s.lease, true, `create table test_add_index (c1 bigint, c2 bigint, c3 bigint, key(c1)) partition by range columns (c3) ( @@ -933,7 +927,7 @@ func (s *testDBSuite1) TestAddIndex1(c *C) { "create table test_add_index (c1 bigint, c2 bigint, c3 bigint, primary key(c1))", "") } -func (s *testDBSuite8) TestAddIndex2(c *C) { +func (s *testDBSuite2) TestAddIndex2(c *C) { testAddIndex(c, s.store, s.lease, true, `create table test_add_index (c1 bigint, c2 bigint, c3 bigint, primary key(c1)) partition by range (c1) ( @@ -950,7 +944,7 @@ func (s *testDBSuite3) TestAddIndex3(c *C) { partition by hash (c1) partitions 4;`, "") } -func (s *testDBSuite8) TestAddIndex4(c *C) { +func (s *testDBSuite4) TestAddIndex4(c *C) { testAddIndex(c, s.store, s.lease, true, `create table test_add_index (c1 bigint, c2 bigint, c3 bigint, primary key(c1)) partition by range columns (c1) ( @@ -1413,7 +1407,7 @@ func checkDelRangeDone(c *C, ctx sessionctx.Context, idx table.Index) { c.Assert(handles, HasLen, 0, Commentf("take time %v", time.Since(startTime))) } -func (s *testDBSuite8) TestAlterPrimaryKey(c *C) { +func (s *testDBSuite5) TestAlterPrimaryKey(c *C) { s.tk = testkit.NewTestKitWithInit(c, s.store) s.tk.MustExec("create table test_add_pk(a int, b int unsigned , c varchar(255) default 'abc', d int as (a+b), e int as (a+1) stored, index idx(b))") defer s.tk.MustExec("drop table test_add_pk") @@ -1547,7 +1541,7 @@ func (s *testDBSuite5) TestCreateIndexType(c *C) { s.tk.MustExec(sql) } -func (s *testDBSuite4) TestColumn(c *C) { +func (s *testDBSuite1) TestColumn(c *C) { s.tk = testkit.NewTestKit(c, s.store) s.tk.MustExec("use " + s.schemaName) s.tk.MustExec("create table t2 (c1 int, c2 int, c3 int)") @@ -1771,7 +1765,7 @@ LOOP: // TestDropColumn is for inserting value with a to-be-dropped column when do drop column. // Column info from schema in build-insert-plan should be public only, // otherwise they will not be consist with Table.Col(), then the server will panic. -func (s *testDBSuite7) TestDropColumn(c *C) { +func (s *testDBSuite2) TestDropColumn(c *C) { s.tk = testkit.NewTestKit(c, s.store) s.tk.MustExec("create database drop_col_db") s.tk.MustExec("use drop_col_db") From 13bf6cc196b5b60999617a8e7ace4cd9040c5e88 Mon Sep 17 00:00:00 2001 From: Shenghui Wu <793703860@qq.com> Date: Fri, 7 Feb 2020 09:39:53 +0800 Subject: [PATCH 4/5] executor: implement disk-based sort (Part 2) (#14279) --- executor/benchmark_test.go | 170 +++++++++++++--- executor/executor_pkg_test.go | 91 +++++++++ executor/executor_test.go | 2 +- executor/sort.go | 369 +++++++++++++++++----------------- executor/sort_test.go | 56 ++++++ util/chunk/row_container.go | 66 ++++-- 6 files changed, 520 insertions(+), 234 deletions(-) create mode 100644 executor/sort_test.go diff --git a/executor/benchmark_test.go b/executor/benchmark_test.go index fdb823849f0d8..d937c2c4f4cfd 100644 --- a/executor/benchmark_test.go +++ b/executor/benchmark_test.go @@ -15,6 +15,7 @@ package executor import ( "context" + "encoding/base64" "fmt" "math/rand" "sort" @@ -46,13 +47,12 @@ var ( ) type mockDataSourceParameters struct { - schema *expression.Schema - genDataFunc func(row int, typ *types.FieldType) interface{} - ndvs []int // number of distinct values on columns[i] and zero represents no limit - orders []bool // columns[i] should be ordered if orders[i] is true - rows int // number of rows the DataSource should output - ctx sessionctx.Context - isRawDataSmall bool // false: rawData, true: rawDataSmall + schema *expression.Schema + genDataFunc func(row int, typ *types.FieldType) interface{} + ndvs []int // number of distinct values on columns[i] and zero represents no limit + orders []bool // columns[i] should be ordered if orders[i] is true + rows int // number of rows the DataSource should output + ctx sessionctx.Context } type mockDataSource struct { @@ -154,10 +154,9 @@ func (mds *mockDataSource) randDatum(typ *types.FieldType) interface{} { case mysql.TypeDouble: return rand.Float64() case mysql.TypeVarString: - if mds.p.isRawDataSmall { - return rawDataSmall - } - return rawData + buff := make([]byte, 10) + rand.Read(buff) + return base64.RawURLEncoding.EncodeToString(buff) default: panic("not implement") } @@ -508,18 +507,14 @@ type windowTestCase struct { concurrency int dataSourceSorted bool ctx sessionctx.Context + rawDataSmall string } -var rawData = strings.Repeat("x", 5*1024) -var rawDataSmall = strings.Repeat("x", 16) - func (a windowTestCase) columns() []*expression.Column { - rawDataTp := new(types.FieldType) - types.DefaultTypeForValue(rawData, rawDataTp) return []*expression.Column{ {Index: 0, RetType: types.NewFieldType(mysql.TypeDouble)}, {Index: 1, RetType: types.NewFieldType(mysql.TypeLonglong)}, - {Index: 2, RetType: rawDataTp}, + {Index: 2, RetType: types.NewFieldType(mysql.TypeVarString)}, {Index: 3, RetType: types.NewFieldType(mysql.TypeLonglong)}, } } @@ -533,7 +528,7 @@ func defaultWindowTestCase() *windowTestCase { ctx := mock.NewContext() ctx.GetSessionVars().InitChunkSize = variable.DefInitChunkSize ctx.GetSessionVars().MaxChunkSize = variable.DefMaxChunkSize - return &windowTestCase{ast.WindowFuncRowNumber, 1, nil, 1000, 10000000, 1, true, ctx} + return &windowTestCase{ast.WindowFuncRowNumber, 1, nil, 1000, 10000000, 1, true, ctx, strings.Repeat("x", 16)} } func benchmarkWindowExecWithCase(b *testing.B, casTest *windowTestCase) { @@ -544,12 +539,21 @@ func benchmarkWindowExecWithCase(b *testing.B, casTest *windowTestCase) { cols := casTest.columns() dataSource := buildMockDataSource(mockDataSourceParameters{ - schema: expression.NewSchema(cols...), - ndvs: []int{0, casTest.ndv, 0, 0}, - orders: []bool{false, casTest.dataSourceSorted, false, false}, - rows: casTest.rows, - ctx: casTest.ctx, - isRawDataSmall: true, + schema: expression.NewSchema(cols...), + ndvs: []int{0, casTest.ndv, 0, 0}, + orders: []bool{false, casTest.dataSourceSorted, false, false}, + rows: casTest.rows, + ctx: casTest.ctx, + genDataFunc: func(row int, typ *types.FieldType) interface{} { + switch typ.Tp { + case mysql.TypeLong, mysql.TypeLonglong: + return int64(row) + case mysql.TypeVarString: + return casTest.rawDataSmall + default: + panic("not implement") + } + }, }) b.ResetTimer() @@ -679,6 +683,7 @@ type hashJoinTestCase struct { joinType core.JoinType disk bool useOuterToBuild bool + rawData string } func (tc hashJoinTestCase) columns() []*expression.Column { @@ -702,7 +707,7 @@ func defaultHashJoinTestCase(cols []*types.FieldType, joinType core.JoinType, us ctx.GetSessionVars().StmtCtx.MemTracker = memory.NewTracker(nil, -1) ctx.GetSessionVars().StmtCtx.DiskTracker = disk.NewTracker(nil, -1) ctx.GetSessionVars().IndexLookupJoinConcurrency = 4 - tc := &hashJoinTestCase{rows: 100000, concurrency: 4, ctx: ctx, keyIdx: []int{0, 1}} + tc := &hashJoinTestCase{rows: 100000, concurrency: 4, ctx: ctx, keyIdx: []int{0, 1}, rawData: strings.Repeat("x", 5*1024)} tc.cols = cols tc.useOuterToBuild = useOuterToBuild tc.joinType = joinType @@ -762,7 +767,7 @@ func benchmarkHashJoinExecWithCase(b *testing.B, casTest *hashJoinTestCase) { case mysql.TypeLong, mysql.TypeLonglong: return int64(row) case mysql.TypeVarString: - return rawData + return casTest.rawData case mysql.TypeDouble: return float64(row) default: @@ -915,7 +920,7 @@ func benchmarkBuildHashTableForList(b *testing.B, casTest *hashJoinTestCase) { case mysql.TypeLong, mysql.TypeLonglong: return int64(row) case mysql.TypeVarString: - return rawData + return casTest.rawData default: panic("not implement") } @@ -994,6 +999,7 @@ type indexJoinTestCase struct { innerJoinKeyIdx []int innerIdx []int needOuterSort bool + rawData string } func (tc indexJoinTestCase) columns() []*expression.Column { @@ -1019,6 +1025,7 @@ func defaultIndexJoinTestCase() *indexJoinTestCase { outerJoinKeyIdx: []int{0, 1}, innerJoinKeyIdx: []int{0, 1}, innerIdx: []int{0, 1}, + rawData: strings.Repeat("x", 5*1024), } return tc } @@ -1039,7 +1046,7 @@ func (tc indexJoinTestCase) getMockDataSourceOptByRows(rows int) mockDataSourceP case mysql.TypeDouble: return float64(row) case mysql.TypeVarString: - return rawData + return tc.rawData default: panic("not implement") } @@ -1316,6 +1323,7 @@ func newMergeJoinBenchmark(numOuterRows, numInnerDup, numInnerRedundant int) (tc outerJoinKeyIdx: []int{0, 1}, innerJoinKeyIdx: []int{0, 1}, innerIdx: []int{0, 1}, + rawData: strings.Repeat("x", 5*1024), } tc = &mergeJoinTestCase{*itc} outerOpt := mockDataSourceParameters{ @@ -1329,7 +1337,7 @@ func newMergeJoinBenchmark(numOuterRows, numInnerDup, numInnerRedundant int) (tc case mysql.TypeDouble: return float64(row) case mysql.TypeVarString: - return rawData + return tc.rawData default: panic("not implement") } @@ -1348,7 +1356,7 @@ func newMergeJoinBenchmark(numOuterRows, numInnerDup, numInnerRedundant int) (tc case mysql.TypeDouble: return float64(row) case mysql.TypeVarString: - return rawData + return tc.rawData default: panic("not implement") } @@ -1443,3 +1451,105 @@ func BenchmarkMergeJoinExec(b *testing.B) { }) } } + +type sortCase struct { + rows int + orderByIdx []int + ndvs []int + ctx sessionctx.Context +} + +func (tc sortCase) columns() []*expression.Column { + return []*expression.Column{ + {Index: 0, RetType: types.NewFieldType(mysql.TypeLonglong)}, + {Index: 1, RetType: types.NewFieldType(mysql.TypeLonglong)}, + } +} + +func (tc sortCase) String() string { + return fmt.Sprintf("(rows:%v, orderBy:%v, ndvs: %v)", tc.rows, tc.orderByIdx, tc.ndvs) +} + +func defaultSortTestCase() *sortCase { + ctx := mock.NewContext() + ctx.GetSessionVars().InitChunkSize = variable.DefInitChunkSize + ctx.GetSessionVars().MaxChunkSize = variable.DefMaxChunkSize + ctx.GetSessionVars().StmtCtx.MemTracker = memory.NewTracker(nil, -1) + tc := &sortCase{rows: 300000, orderByIdx: []int{0, 1}, ndvs: []int{0, 0}, ctx: ctx} + return tc +} + +func benchmarkSortExec(b *testing.B, cas *sortCase) { + opt := mockDataSourceParameters{ + schema: expression.NewSchema(cas.columns()...), + rows: cas.rows, + ctx: cas.ctx, + ndvs: cas.ndvs, + } + dataSource := buildMockDataSource(opt) + exec := &SortExec{ + baseExecutor: newBaseExecutor(cas.ctx, dataSource.schema, stringutil.StringerStr("sort"), dataSource), + ByItems: make([]*core.ByItems, 0, len(cas.orderByIdx)), + schema: dataSource.schema, + } + for _, idx := range cas.orderByIdx { + exec.ByItems = append(exec.ByItems, &core.ByItems{Expr: cas.columns()[idx]}) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + b.StopTimer() + tmpCtx := context.Background() + chk := newFirstChunk(exec) + dataSource.prepareChunks() + + b.StartTimer() + if err := exec.Open(tmpCtx); err != nil { + b.Fatal(err) + } + for { + if err := exec.Next(tmpCtx, chk); err != nil { + b.Fatal(err) + } + if chk.NumRows() == 0 { + break + } + } + + if err := exec.Close(); err != nil { + b.Fatal(err) + } + b.StopTimer() + } +} + +func BenchmarkSortExec(b *testing.B) { + b.ReportAllocs() + cas := defaultSortTestCase() + // all random data + cas.ndvs = []int{0, 0} + cas.orderByIdx = []int{0, 1} + b.Run(fmt.Sprintf("%v", cas), func(b *testing.B) { + benchmarkSortExec(b, cas) + }) + + ndvs := []int{1, 10000} + for _, ndv := range ndvs { + cas.ndvs = []int{ndv, 0} + cas.orderByIdx = []int{0, 1} + b.Run(fmt.Sprintf("%v", cas), func(b *testing.B) { + benchmarkSortExec(b, cas) + }) + + cas.ndvs = []int{ndv, 0} + cas.orderByIdx = []int{0} + b.Run(fmt.Sprintf("%v", cas), func(b *testing.B) { + benchmarkSortExec(b, cas) + }) + + cas.ndvs = []int{ndv, 0} + cas.orderByIdx = []int{1} + b.Run(fmt.Sprintf("%v", cas), func(b *testing.B) { + benchmarkSortExec(b, cas) + }) + } +} diff --git a/executor/executor_pkg_test.go b/executor/executor_pkg_test.go index 904abacb1c7f2..f0873875d1255 100644 --- a/executor/executor_pkg_test.go +++ b/executor/executor_pkg_test.go @@ -20,12 +20,17 @@ import ( "github.com/pingcap/parser/ast" "github.com/pingcap/parser/auth" "github.com/pingcap/parser/mysql" + "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/expression" + "github.com/pingcap/tidb/planner/core" + "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/memory" "github.com/pingcap/tidb/util/mock" "github.com/pingcap/tidb/util/ranger" + "github.com/pingcap/tidb/util/stringutil" ) var _ = Suite(&testExecSuite{}) @@ -234,3 +239,89 @@ func assertEqualStrings(c *C, got []field, expect []string) { c.Assert(string(got[i].str), Equals, expect[i]) } } + +func (s *testExecSuite) TestSortSpillDisk(c *C) { + originCfg := config.GetGlobalConfig() + newConf := *originCfg + newConf.OOMUseTmpStorage = true + newConf.MemQuotaQuery = 1 + config.StoreGlobalConfig(&newConf) + defer config.StoreGlobalConfig(originCfg) + + ctx := mock.NewContext() + ctx.GetSessionVars().InitChunkSize = variable.DefMaxChunkSize + ctx.GetSessionVars().MaxChunkSize = variable.DefMaxChunkSize + ctx.GetSessionVars().StmtCtx.MemTracker = memory.NewTracker(nil, -1) + cas := &sortCase{rows: 2048, orderByIdx: []int{0, 1}, ndvs: []int{0, 0}, ctx: ctx} + opt := mockDataSourceParameters{ + schema: expression.NewSchema(cas.columns()...), + rows: cas.rows, + ctx: cas.ctx, + ndvs: cas.ndvs, + } + dataSource := buildMockDataSource(opt) + exec := &SortExec{ + baseExecutor: newBaseExecutor(cas.ctx, dataSource.schema, stringutil.StringerStr("sort"), dataSource), + ByItems: make([]*core.ByItems, 0, len(cas.orderByIdx)), + schema: dataSource.schema, + } + for _, idx := range cas.orderByIdx { + exec.ByItems = append(exec.ByItems, &core.ByItems{Expr: cas.columns()[idx]}) + } + tmpCtx := context.Background() + chk := newFirstChunk(exec) + dataSource.prepareChunks() + err := exec.Open(tmpCtx) + c.Assert(err, IsNil) + for { + err = exec.Next(tmpCtx, chk) + c.Assert(err, IsNil) + if chk.NumRows() == 0 { + break + } + } + // Test only 1 partition and all data in memory. + c.Assert(len(exec.partitionList), Equals, 1) + c.Assert(exec.partitionList[0].AlreadySpilled(), Equals, false) + c.Assert(exec.partitionList[0].NumRow(), Equals, 2048) + err = exec.Close() + c.Assert(err, IsNil) + + ctx.GetSessionVars().StmtCtx.MemTracker = memory.NewTracker(nil, 1) + dataSource.prepareChunks() + err = exec.Open(tmpCtx) + c.Assert(err, IsNil) + for { + err = exec.Next(tmpCtx, chk) + c.Assert(err, IsNil) + if chk.NumRows() == 0 { + break + } + } + // Test 2 partitions and all data in disk. + c.Assert(len(exec.partitionList), Equals, 2) + c.Assert(exec.partitionList[0].AlreadySpilled(), Equals, true) + c.Assert(exec.partitionList[1].AlreadySpilled(), Equals, true) + c.Assert(exec.partitionList[0].NumRow(), Equals, 1024) + c.Assert(exec.partitionList[1].NumRow(), Equals, 1024) + err = exec.Close() + c.Assert(err, IsNil) + + ctx.GetSessionVars().StmtCtx.MemTracker = memory.NewTracker(nil, 24000) + dataSource.prepareChunks() + err = exec.Open(tmpCtx) + c.Assert(err, IsNil) + for { + err = exec.Next(tmpCtx, chk) + c.Assert(err, IsNil) + if chk.NumRows() == 0 { + break + } + } + // Test only 1 partition but spill disk. + c.Assert(len(exec.partitionList), Equals, 1) + c.Assert(exec.partitionList[0].AlreadySpilled(), Equals, true) + c.Assert(exec.partitionList[0].NumRow(), Equals, 2048) + err = exec.Close() + c.Assert(err, IsNil) +} diff --git a/executor/executor_test.go b/executor/executor_test.go index 909b84a207cec..28a4ab5621f48 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -4446,7 +4446,7 @@ func (s *testSuite) TestOOMPanicAction(c *C) { c.Assert(err.Error(), Matches, "Out Of Memory Quota!.*") tk.MustExec("set @@tidb_mem_quota_query=10000") tk.MustExec("insert into t1 values (1),(2),(3),(4),(5);") - tk.MustExec("set @@tidb_mem_quota_query=200;") + tk.MustExec("set @@tidb_mem_quota_query=10;") _, err = tk.Exec("insert into t select a from t1 order by a desc;") c.Assert(err.Error(), Matches, "Out Of Memory Quota!.*") _, err = tk.Exec("replace into t select a from t1 order by a desc;") diff --git a/executor/sort.go b/executor/sort.go index 65129c8c7f8c9..b38470c27b808 100644 --- a/executor/sort.go +++ b/executor/sort.go @@ -18,8 +18,8 @@ import ( "context" "fmt" "sort" - "sync/atomic" + "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/expression" plannercore "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/types" @@ -47,61 +47,52 @@ type SortExec struct { // keyCmpFuncs is used to compare each ByItem. keyCmpFuncs []chunk.CompareFunc // rowChunks is the chunks to store row values. - rowChunks *chunk.List + rowChunks *chunk.RowContainer // rowPointer store the chunk index and row index for each row. rowPtrs []chunk.RowPtr memTracker *memory.Tracker diskTracker *disk.Tracker - // rowChunksInDisk is the chunks to store row values in disk. - rowChunksInDisk *chunk.ListInDisk - // rowPtrsInDisk store the disk-chunk index and row index for each row. - rowPtrsInDisk []chunk.RowPtr - // partitionList is the chunks to store row values in disk for partitions. - partitionList []*chunk.ListInDisk - // partitionRowPtrs store the disk-chunk index and row index for each row for partitions. + // partitionList is the chunks to store row values for partitions. + partitionList []*chunk.RowContainer + // partitionRowPtrs store the sorted RowPtrs for each row for partitions. partitionRowPtrs [][]chunk.RowPtr - // exceeded indicates that records have exceeded memQuota during - // adding this chunk and we should spill now. - exceeded uint32 - // spilled indicates that records have spilled out into disk. - spilled uint32 + + // multiWayMerge uses multi-way merge for spill disk. + // The multi-way merge algorithm can refer to https://en.wikipedia.org/wiki/K-way_merge_algorithm + multiWayMerge *multiWayMerge + // spillAction save the Action for spill disk. + spillAction *chunk.SpillDiskAction } // Close implements the Executor Close interface. func (e *SortExec) Close() error { - if e.alreadySpilled() { - if e.rowChunksInDisk != nil { - if err := e.rowChunksInDisk.Close(); err != nil { + for _, chunkInDisk := range e.partitionList { + if chunkInDisk != nil { + if err := chunkInDisk.Close(); err != nil { return err } } - for _, chunkInDisk := range e.partitionList { - if chunkInDisk != nil { - if err := chunkInDisk.Close(); err != nil { - return err - } - } - } - e.rowChunksInDisk = nil - e.partitionList = e.partitionList[:0] + } + e.partitionList = e.partitionList[:0] - e.memTracker.Consume(int64(-8 * cap(e.rowPtrsInDisk))) - e.rowPtrsInDisk = nil - for _, partitionPtrs := range e.partitionRowPtrs { - e.memTracker.Consume(int64(-8 * cap(partitionPtrs))) + for _, chunkPtr := range e.partitionRowPtrs { + if chunkPtr != nil { + e.memTracker.Consume(int64(-8 * cap(chunkPtr))) } - e.partitionRowPtrs = nil } + e.partitionRowPtrs = e.partitionRowPtrs[:0] + if e.rowChunks != nil { e.memTracker.Consume(-e.rowChunks.GetMemTracker().BytesConsumed()) e.rowChunks = nil } - e.memTracker.Consume(int64(-8 * cap(e.rowPtrs))) e.rowPtrs = nil e.memTracker = nil e.diskTracker = nil + e.multiWayMerge = nil + e.spillAction = nil return e.children[0].Close() } @@ -117,87 +108,148 @@ func (e *SortExec) Open(ctx context.Context) error { e.diskTracker = memory.NewTracker(e.id, -1) e.diskTracker.AttachTo(e.ctx.GetSessionVars().StmtCtx.DiskTracker) } - e.exceeded = 0 - e.spilled = 0 - e.rowChunksInDisk = nil - e.rowPtrsInDisk = e.rowPtrsInDisk[:0] e.partitionList = e.partitionList[:0] - e.partitionRowPtrs = e.partitionRowPtrs[:0] return e.children[0].Open(ctx) } // Next implements the Executor Next interface. +// Sort constructs the result following these step: +// 1. Read as mush as rows into memory. +// 2. If memory quota is triggered, sort these rows in memory and put them into disk as partition 1, then reset +// the memory quota trigger and return to step 1 +// 3. If memory quota is not triggered and child is consumed, sort these rows in memory as partition N. +// 4. Merge sort if the count of partitions is larger than 1. If there is only one partition in step 4, it works +// just like in-memory sort before. func (e *SortExec) Next(ctx context.Context, req *chunk.Chunk) error { req.Reset() if !e.fetched { + e.initCompareFuncs() + e.buildKeyColumns() err := e.fetchRowChunks(ctx) if err != nil { return err } - if e.alreadySpilled() { - err = e.prepareExternalSorting() - if err != nil { - return err - } - e.fetched = true - } else { - e.initPointers() - e.initCompareFuncs() - e.buildKeyColumns() - sort.Slice(e.rowPtrs, e.keyColumnsLess) - e.fetched = true - } + e.fetched = true } - if e.alreadySpilled() { - for !req.IsFull() && e.Idx < len(e.partitionRowPtrs[0]) { - rowPtr := e.partitionRowPtrs[0][e.Idx] - row, err := e.partitionList[0].GetRow(rowPtr) - if err != nil { - return err - } - req.AppendRow(row) - e.Idx++ + if len(e.partitionList) == 0 { + return nil + } + if len(e.partitionList) > 1 { + if err := e.externalSorting(req); err != nil { + return err } } else { - for !req.IsFull() && e.Idx < len(e.rowPtrs) { - rowPtr := e.rowPtrs[e.Idx] - req.AppendRow(e.rowChunks.GetRow(rowPtr)) - e.Idx++ + // Check whether the one partition is spilled. + // If the partition is in memory, use List.GetRow() to get better performance. + if !e.partitionList[0].AlreadySpilled() { + rowChunks := e.partitionList[0].GetList() + for !req.IsFull() && e.Idx < len(e.rowPtrs) { + rowPtr := e.partitionRowPtrs[0][e.Idx] + req.AppendRow(rowChunks.GetRow(rowPtr)) + e.Idx++ + } + } else { + for !req.IsFull() && e.Idx < len(e.rowPtrs) { + rowPtr := e.partitionRowPtrs[0][e.Idx] + row, err := e.partitionList[0].GetRow(rowPtr) + if err != nil { + return err + } + req.AppendRow(row) + e.Idx++ + } } } return nil } -func (e *SortExec) prepareExternalSorting() (err error) { - e.initCompareFuncs() - e.buildKeyColumns() - e.rowPtrsInDisk = e.initPointersForListInDisk(e.rowChunksInDisk) - // partition sort - // Now only have one partition. - // The partition will be adjusted in the next pr. - err = e.readPartition(e.rowChunksInDisk, e.rowPtrsInDisk) - if err != nil { - return err - } +func (e *SortExec) generatePartition() { e.initPointers() sort.Slice(e.rowPtrs, e.keyColumnsLess) - listInDisk, err := e.spillToDiskByRowPtr() - if err != nil { - return err + e.partitionList = append(e.partitionList, e.rowChunks) + e.partitionRowPtrs = append(e.partitionRowPtrs, e.rowPtrs) +} + +type partitionPointer struct { + row chunk.Row + partitionID int + consumed int +} + +type multiWayMerge struct { + lessRowFunction func(rowI chunk.Row, rowJ chunk.Row) bool + elements []partitionPointer +} + +func (h *multiWayMerge) Less(i, j int) bool { + rowI := h.elements[i].row + rowJ := h.elements[j].row + return h.lessRowFunction(rowI, rowJ) +} + +func (h *multiWayMerge) Len() int { + return len(h.elements) +} + +func (h *multiWayMerge) Push(x interface{}) { + // Should never be called. +} + +func (h *multiWayMerge) Pop() interface{} { + h.elements = h.elements[:len(h.elements)-1] + return nil +} + +func (h *multiWayMerge) Swap(i, j int) { + h.elements[i], h.elements[j] = h.elements[j], h.elements[i] +} + +func (e *SortExec) externalSorting(req *chunk.Chunk) (err error) { + if e.multiWayMerge == nil { + e.multiWayMerge = &multiWayMerge{e.lessRow, make([]partitionPointer, 0, len(e.partitionList))} + for i := 0; i < len(e.partitionList); i++ { + row, err := e.partitionList[i].GetRow(e.partitionRowPtrs[i][0]) + if err != nil { + return err + } + e.multiWayMerge.elements = append(e.multiWayMerge.elements, partitionPointer{row: row, partitionID: i, consumed: 0}) + } + heap.Init(e.multiWayMerge) } - e.memTracker.Consume(-e.rowChunks.GetMemTracker().BytesConsumed()) - e.rowChunks = nil - e.partitionList = append(e.partitionList, listInDisk) - e.partitionRowPtrs = append(e.partitionRowPtrs, e.initPointersForListInDisk(listInDisk)) - return err + + for !req.IsFull() && e.multiWayMerge.Len() > 0 { + partitionPtr := e.multiWayMerge.elements[0] + req.AppendRow(partitionPtr.row) + partitionPtr.consumed++ + if partitionPtr.consumed >= len(e.partitionRowPtrs[partitionPtr.partitionID]) { + heap.Remove(e.multiWayMerge, 0) + continue + } + partitionPtr.row, err = e.partitionList[partitionPtr.partitionID].GetRow(e.partitionRowPtrs[partitionPtr.partitionID][partitionPtr.consumed]) + if err != nil { + return err + } + e.multiWayMerge.elements[0] = partitionPtr + heap.Fix(e.multiWayMerge, 0) + } + return nil } func (e *SortExec) fetchRowChunks(ctx context.Context) error { fields := retTypes(e) - e.rowChunks = chunk.NewList(fields, e.initCap, e.maxChunkSize) + e.rowChunks = chunk.NewRowContainer(fields, e.maxChunkSize) e.rowChunks.GetMemTracker().AttachTo(e.memTracker) e.rowChunks.GetMemTracker().SetLabel(rowChunksLabel) + var onExceededCallback func(rowContainer *chunk.RowContainer) + if config.GetGlobalConfig().OOMUseTmpStorage { + e.spillAction = e.rowChunks.ActionSpill() + e.ctx.GetSessionVars().StmtCtx.MemTracker.FallbackOldAndSetNewAction(e.spillAction) + onExceededCallback = func(rowContainer *chunk.RowContainer) { + e.generatePartition() + } + e.rowChunks.SetOnExceededCallback(onExceededCallback) + } for { chk := newFirstChunk(e.children[0]) err := Next(ctx, e.children[0], chk) @@ -208,35 +260,25 @@ func (e *SortExec) fetchRowChunks(ctx context.Context) error { if rowCount == 0 { break } - if e.alreadySpilled() { - // append chk to disk. - err := e.rowChunksInDisk.Add(chk) - if err != nil { - return err - } - } else { - e.rowChunks.Add(chk) - if atomic.LoadUint32(&e.exceeded) == 1 { - e.rowChunksInDisk, err = e.spillToDisk() - if err != nil { - return err - } - e.memTracker.Consume(-e.rowChunks.GetMemTracker().BytesConsumed()) - e.rowChunks = nil // GC its internal chunks. - atomic.StoreUint32(&e.spilled, 1) - } + if err := e.rowChunks.Add(chk); err != nil { + return err } + if e.rowChunks.AlreadySpilled() { + e.rowChunks = chunk.NewRowContainer(retTypes(e), e.maxChunkSize) + e.rowChunks.GetMemTracker().AttachTo(e.memTracker) + e.rowChunks.GetMemTracker().SetLabel(rowChunksLabel) + e.rowChunks.SetOnExceededCallback(onExceededCallback) + e.spillAction.ResetOnceAndSetRowContainer(e.rowChunks) + } + } + if e.rowChunks.NumRow() > 0 { + e.generatePartition() } return nil } func (e *SortExec) initPointers() { - if e.rowPtrs != nil { - e.memTracker.Consume(int64(-8 * cap(e.rowPtrs))) - e.rowPtrs = e.rowPtrs[:0] - } else { - e.rowPtrs = make([]chunk.RowPtr, 0, e.rowChunks.Len()) - } + e.rowPtrs = make([]chunk.RowPtr, 0, e.rowChunks.NumRow()) for chkIdx := 0; chkIdx < e.rowChunks.NumChunks(); chkIdx++ { rowChk := e.rowChunks.GetChunk(chkIdx) for rowIdx := 0; rowIdx < rowChk.NumRows(); rowIdx++ { @@ -246,17 +288,6 @@ func (e *SortExec) initPointers() { e.memTracker.Consume(int64(8 * cap(e.rowPtrs))) } -func (e *SortExec) initPointersForListInDisk(disk *chunk.ListInDisk) []chunk.RowPtr { - rowPtrsInDisk := make([]chunk.RowPtr, 0) - for chkIdx := 0; chkIdx < disk.NumChunks(); chkIdx++ { - for rowIdx := 0; rowIdx < disk.NumRowsOfChunk(chkIdx); rowIdx++ { - rowPtrsInDisk = append(rowPtrsInDisk, chunk.RowPtr{ChkIdx: uint32(chkIdx), RowIdx: uint32(rowIdx)}) - } - } - e.memTracker.Consume(int64(8 * len(rowPtrsInDisk))) - return rowPtrsInDisk -} - func (e *SortExec) initCompareFuncs() { e.keyCmpFuncs = make([]chunk.CompareFunc, len(e.ByItems)) for i := range e.ByItems { @@ -291,67 +322,11 @@ func (e *SortExec) lessRow(rowI, rowJ chunk.Row) bool { // keyColumnsLess is the less function for key columns. func (e *SortExec) keyColumnsLess(i, j int) bool { - rowI := e.rowChunks.GetRow(e.rowPtrs[i]) - rowJ := e.rowChunks.GetRow(e.rowPtrs[j]) + rowI := e.rowChunks.GetList().GetRow(e.rowPtrs[i]) + rowJ := e.rowChunks.GetList().GetRow(e.rowPtrs[j]) return e.lessRow(rowI, rowJ) } -func (e *SortExec) readPartition(disk *chunk.ListInDisk, rowPtrs []chunk.RowPtr) error { - e.rowChunks = chunk.NewList(retTypes(e), e.initCap, e.maxChunkSize) - e.rowChunks.GetMemTracker().AttachTo(e.memTracker) - e.rowChunks.GetMemTracker().SetLabel(rowChunksLabel) - for _, rowPtr := range rowPtrs { - rowPtr, err := disk.GetRow(rowPtr) - if err != nil { - return err - } - e.rowChunks.AppendRow(rowPtr) - } - return nil -} - -// alreadySpilled indicates that records have spilled out into disk. -func (e *SortExec) alreadySpilled() bool { return e.rowChunksInDisk != nil } - -// alreadySpilledSafe indicates that records have spilled out into disk. It's thread-safe. -func (e *SortExec) alreadySpilledSafe() bool { return atomic.LoadUint32(&e.spilled) == 1 } - -func (e *SortExec) spillToDisk() (disk *chunk.ListInDisk, err error) { - N := e.rowChunks.NumChunks() - rowChunksInDisk := chunk.NewListInDisk(e.retFieldTypes) - rowChunksInDisk.GetDiskTracker().AttachTo(e.diskTracker) - for i := 0; i < N; i++ { - chk := e.rowChunks.GetChunk(i) - err = rowChunksInDisk.Add(chk) - if err != nil { - return nil, err - } - } - return rowChunksInDisk, nil -} - -func (e *SortExec) spillToDiskByRowPtr() (disk *chunk.ListInDisk, err error) { - rowChunksInDisk := chunk.NewListInDisk(e.retFieldTypes) - rowChunksInDisk.GetDiskTracker().AttachTo(e.diskTracker) - chk := newFirstChunk(e) - for _, rowPtr := range e.rowPtrs { - chk.AppendRow(e.rowChunks.GetRow(rowPtr)) - if chk.IsFull() { - err := rowChunksInDisk.Add(chk) - if err != nil { - return nil, err - } - chk = newFirstChunk(e) - } - } - if chk.NumRows() != 0 { - if err := rowChunksInDisk.Add(chk); err != nil { - return nil, err - } - } - return rowChunksInDisk, nil -} - // TopNExec implements a Top-N algorithm and it is built from a SELECT statement with ORDER BY and LIMIT. // Instead of sorting all the rows fetched from the table, it keeps the Top-N elements only in a heap to reduce memory usage. type TopNExec struct { @@ -370,8 +345,8 @@ type topNChunkHeap struct { // Less implement heap.Interface, but since we mantains a max heap, // this function returns true if row i is greater than row j. func (h *topNChunkHeap) Less(i, j int) bool { - rowI := h.rowChunks.GetRow(h.rowPtrs[i]) - rowJ := h.rowChunks.GetRow(h.rowPtrs[j]) + rowI := h.rowChunks.GetList().GetRow(h.rowPtrs[i]) + rowJ := h.rowChunks.GetList().GetRow(h.rowPtrs[j]) return h.greaterRow(rowI, rowJ) } @@ -436,7 +411,10 @@ func (e *TopNExec) Next(ctx context.Context, req *chunk.Chunk) error { return nil } for !req.IsFull() && e.Idx < len(e.rowPtrs) { - row := e.rowChunks.GetRow(e.rowPtrs[e.Idx]) + row, err := e.rowChunks.GetRow(e.rowPtrs[e.Idx]) + if err != nil { + return err + } req.AppendRow(row) e.Idx++ } @@ -445,13 +423,13 @@ func (e *TopNExec) Next(ctx context.Context, req *chunk.Chunk) error { func (e *TopNExec) loadChunksUntilTotalLimit(ctx context.Context) error { e.chkHeap = &topNChunkHeap{e} - e.rowChunks = chunk.NewList(retTypes(e), e.initCap, e.maxChunkSize) + e.rowChunks = chunk.NewRowContainer(retTypes(e), e.maxChunkSize) e.rowChunks.GetMemTracker().AttachTo(e.memTracker) e.rowChunks.GetMemTracker().SetLabel(rowChunksLabel) - for uint64(e.rowChunks.Len()) < e.totalLimit { + for uint64(e.rowChunks.NumRow()) < e.totalLimit { srcChk := newFirstChunk(e.children[0]) // adjust required rows by total limit - srcChk.SetRequiredRows(int(e.totalLimit-uint64(e.rowChunks.Len())), e.maxChunkSize) + srcChk.SetRequiredRows(int(e.totalLimit-uint64(e.rowChunks.NumRow())), e.maxChunkSize) err := Next(ctx, e.children[0], srcChk) if err != nil { return err @@ -459,7 +437,9 @@ func (e *TopNExec) loadChunksUntilTotalLimit(ctx context.Context) error { if srcChk.NumRows() == 0 { break } - e.rowChunks.Add(srcChk) + if err := e.rowChunks.Add(srcChk); err != nil { + return err + } } e.initPointers() e.initCompareFuncs() @@ -488,7 +468,7 @@ func (e *TopNExec) executeTopN(ctx context.Context) error { if err != nil { return err } - if e.rowChunks.Len() > len(e.rowPtrs)*topNCompactionFactor { + if e.rowChunks.NumRow() > len(e.rowPtrs)*topNCompactionFactor { err = e.doCompaction() if err != nil { return err @@ -503,11 +483,17 @@ func (e *TopNExec) processChildChk(childRowChk *chunk.Chunk) error { for i := 0; i < childRowChk.NumRows(); i++ { heapMaxPtr := e.rowPtrs[0] var heapMax, next chunk.Row - heapMax = e.rowChunks.GetRow(heapMaxPtr) + heapMax, err := e.rowChunks.GetRow(heapMaxPtr) + if err != nil { + return err + } next = childRowChk.GetRow(i) if e.chkHeap.greaterRow(heapMax, next) { // Evict heap max, keep the next row. - e.rowPtrs[0] = e.rowChunks.AppendRow(childRowChk.GetRow(i)) + e.rowPtrs[0], err = e.rowChunks.AppendRow(childRowChk.GetRow(i)) + if err != nil { + return err + } heap.Fix(e.chkHeap, 0) } } @@ -519,10 +505,17 @@ func (e *TopNExec) processChildChk(childRowChk *chunk.Chunk) error { // but we want descending top N, then we will keep all data in memory. // But if data is distributed randomly, this function will be called log(n) times. func (e *TopNExec) doCompaction() error { - newRowChunks := chunk.NewList(retTypes(e), e.initCap, e.maxChunkSize) - newRowPtrs := make([]chunk.RowPtr, 0, e.rowChunks.Len()) + newRowChunks := chunk.NewRowContainer(retTypes(e), e.maxChunkSize) + newRowPtrs := make([]chunk.RowPtr, 0, e.rowChunks.NumRow()) for _, rowPtr := range e.rowPtrs { - newRowPtr := newRowChunks.AppendRow(e.rowChunks.GetRow(rowPtr)) + row, err := e.rowChunks.GetRow(rowPtr) + if err != nil { + return err + } + newRowPtr, err := newRowChunks.AppendRow(row) + if err != nil { + return err + } newRowPtrs = append(newRowPtrs, newRowPtr) } newRowChunks.GetMemTracker().SetLabel(rowChunksLabel) diff --git a/executor/sort_test.go b/executor/sort_test.go new file mode 100644 index 0000000000000..4f2ccd5e0ff36 --- /dev/null +++ b/executor/sort_test.go @@ -0,0 +1,56 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package executor_test + +import ( + "fmt" + + . "github.com/pingcap/check" + "github.com/pingcap/tidb/config" + "github.com/pingcap/tidb/util" + "github.com/pingcap/tidb/util/testkit" +) + +func (s *testSuite) TestSortInDisk(c *C) { + originCfg := config.GetGlobalConfig() + newConf := *originCfg + newConf.OOMUseTmpStorage = true + config.StoreGlobalConfig(&newConf) + defer config.StoreGlobalConfig(originCfg) + + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + + sm := &mockSessionManager1{ + PS: make([]*util.ProcessInfo, 0), + } + tk.Se.SetSessionManager(sm) + s.domain.ExpensiveQueryHandle().SetSessionManager(sm) + + tk.MustExec("set @@tidb_mem_quota_query=1;") + tk.MustExec("set @@tidb_max_chunk_size=32;") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(c1 int, c2 int, c3 int)") + for i := 0; i < 5; i++ { + for j := i; j < 1024; j += 5 { + tk.MustExec(fmt.Sprintf("insert into t values(%v, %v, %v)", j, j, j)) + } + } + result := tk.MustQuery("select * from t order by c1") + for i := 0; i < 1024; i++ { + c.Assert(result.Rows()[i][0].(string), Equals, fmt.Sprint(i)) + c.Assert(result.Rows()[i][1].(string), Equals, fmt.Sprint(i)) + c.Assert(result.Rows()[i][2].(string), Equals, fmt.Sprint(i)) + } +} diff --git a/util/chunk/row_container.go b/util/chunk/row_container.go index 09463b052a0ec..7cd2076c49189 100644 --- a/util/chunk/row_container.go +++ b/util/chunk/row_container.go @@ -14,6 +14,7 @@ package chunk import ( + "errors" "sync" "sync/atomic" @@ -44,9 +45,10 @@ type RowContainer struct { // It's for concurrency usage, so access it with atomic. spilled uint32 - memTracker *memory.Tracker - diskTracker *disk.Tracker - actionSpill *spillDiskAction + memTracker *memory.Tracker + diskTracker *disk.Tracker + actionSpill *SpillDiskAction + onExceededCallback func(rowContainer *RowContainer) } // NewRowContainer creates a new RowContainer in memory. @@ -83,7 +85,7 @@ func (c *RowContainer) Reset() error { } atomic.StoreUint32(&c.exceeded, 0) atomic.StoreUint32(&c.spilled, 0) - c.actionSpill.reset() + c.actionSpill.ResetOnce() } else { c.records.Reset() } @@ -127,6 +129,9 @@ func (c *RowContainer) Add(chk *Chunk) (err error) { } else { c.records.Add(chk) if atomic.LoadUint32(&c.exceeded) != 0 { + if c.onExceededCallback != nil { + c.onExceededCallback(c) + } err = c.spillToDisk() if err != nil { return err @@ -137,6 +142,14 @@ func (c *RowContainer) Add(chk *Chunk) (err error) { return } +// AppendRow appends a row to the RowContainer, the row is copied to the RowContainer. +func (c *RowContainer) AppendRow(row Row) (RowPtr, error) { + if c.AlreadySpilled() { + return RowPtr{}, errors.New("ListInDisk don't support AppendRow") + } + return c.records.AppendRow(row), nil +} + // AllocChunk allocates a new chunk from RowContainer. func (c *RowContainer) AllocChunk() (chk *Chunk) { return c.records.allocChunk() @@ -147,6 +160,11 @@ func (c *RowContainer) GetChunk(chkIdx int) *Chunk { return c.records.GetChunk(chkIdx) } +// GetList returns the list of in memory records. +func (c *RowContainer) GetList() *List { + return c.records +} + // GetRow returns the row the ptr pointed to. func (c *RowContainer) GetRow(ptr RowPtr) (Row, error) { if c.AlreadySpilled() { @@ -175,24 +193,32 @@ func (c *RowContainer) Close() (err error) { return } -// ActionSpill returns a memory.ActionOnExceed for spilling over to disk. -func (c *RowContainer) ActionSpill() memory.ActionOnExceed { - c.actionSpill = &spillDiskAction{c: c} +// ActionSpill returns a SpillDiskAction for spilling over to disk. +func (c *RowContainer) ActionSpill() *SpillDiskAction { + c.actionSpill = &SpillDiskAction{c: c} return c.actionSpill } -// spillDiskAction implements memory.ActionOnExceed for chunk.List. If -// the memory quota of a query is exceeded, spillDiskAction.Action is +// SetOnExceededCallback set a callback function for exceeded memory limit. +func (c *RowContainer) SetOnExceededCallback(f func(rowContainer *RowContainer)) { + c.onExceededCallback = f +} + +// SpillDiskAction implements memory.ActionOnExceed for chunk.List. If +// the memory quota of a query is exceeded, SpillDiskAction.Action is // triggered. -type spillDiskAction struct { +type SpillDiskAction struct { once sync.Once c *RowContainer fallbackAction memory.ActionOnExceed + m sync.Mutex } // Action sends a signal to trigger spillToDisk method of RowContainer // and if it is already triggered before, call its fallbackAction. -func (a *spillDiskAction) Action(t *memory.Tracker) { +func (a *SpillDiskAction) Action(t *memory.Tracker) { + a.m.Lock() + defer a.m.Unlock() if a.c.AlreadySpilledSafe() { if a.fallbackAction != nil { a.fallbackAction.Action(t) @@ -205,14 +231,24 @@ func (a *spillDiskAction) Action(t *memory.Tracker) { } // SetFallback sets the fallback action. -func (a *spillDiskAction) SetFallback(fallback memory.ActionOnExceed) { +func (a *SpillDiskAction) SetFallback(fallback memory.ActionOnExceed) { a.fallbackAction = fallback } // SetLogHook sets the hook, it does nothing just to form the memory.ActionOnExceed interface. -func (a *spillDiskAction) SetLogHook(hook func(uint64)) {} +func (a *SpillDiskAction) SetLogHook(hook func(uint64)) {} + +// ResetOnce resets the spill action so that it can be triggered next time. +func (a *SpillDiskAction) ResetOnce() { + a.m.Lock() + defer a.m.Unlock() + a.once = sync.Once{} +} -// reset resets the spill action so that it can be triggered next time. -func (a *spillDiskAction) reset() { +// ResetOnceAndSetRowContainer resets the spill action and sets the RowContainer for the SpillDiskAction. +func (a *SpillDiskAction) ResetOnceAndSetRowContainer(c *RowContainer) { + a.m.Lock() + defer a.m.Unlock() a.once = sync.Once{} + a.c = c } From fd97efefdec176da4b6173bd7a5e0ee9306046a9 Mon Sep 17 00:00:00 2001 From: Chengpeng Yan <41809508+Reminiscent@users.noreply.github.com> Date: Fri, 7 Feb 2020 10:54:53 +0800 Subject: [PATCH 5/5] planner/cascades: add transformation rule MergeAdjacentTopN (#14345) --- planner/cascades/integration_test.go | 24 +++ .../testdata/integration_suite_in.json | 12 ++ .../testdata/integration_suite_out.json | 92 +++++++++++ .../transformation_rules_suite_in.json | 17 +++ .../transformation_rules_suite_out.json | 143 ++++++++++++++++++ planner/cascades/transformation_rules.go | 61 ++++++++ planner/cascades/transformation_rules_test.go | 25 +++ planner/core/logical_plan_builder.go | 5 + 8 files changed, 379 insertions(+) diff --git a/planner/cascades/integration_test.go b/planner/cascades/integration_test.go index 7ec3a74dd86d3..6bbb0cb8aa7d8 100644 --- a/planner/cascades/integration_test.go +++ b/planner/cascades/integration_test.go @@ -268,3 +268,27 @@ func (s *testIntegrationSuite) TestMemTableScan(c *C) { tk.MustQuery(sql).Check(testkit.Rows(output[i].Result...)) } } + +func (s *testIntegrationSuite) TestTopN(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t(a int primary key, b int);") + tk.MustExec("insert into t values (1, 11), (4, 44), (2, 22), (3, 33);") + tk.MustExec("set session tidb_enable_cascades_planner = 1;") + var input []string + var output []struct { + SQL string + Plan []string + Result []string + } + s.testData.GetTestCases(c, &input, &output) + for i, sql := range input { + s.testData.OnRecord(func() { + output[i].SQL = sql + output[i].Plan = s.testData.ConvertRowsToStrings(tk.MustQuery("explain " + sql).Rows()) + output[i].Result = s.testData.ConvertRowsToStrings(tk.MustQuery(sql).Rows()) + }) + tk.MustQuery("explain " + sql).Check(testkit.Rows(output[i].Plan...)) + tk.MustQuery(sql).Check(testkit.Rows(output[i].Result...)) + } +} diff --git a/planner/cascades/testdata/integration_suite_in.json b/planner/cascades/testdata/integration_suite_in.json index aec9d912f423c..3ea30d6ba5a02 100644 --- a/planner/cascades/testdata/integration_suite_in.json +++ b/planner/cascades/testdata/integration_suite_in.json @@ -78,5 +78,17 @@ "cases": [ "select * from information_schema.processlist" ] + }, + { + "name": "TestTopN", + "cases": [ + "select a from (select a from t where b > 2 order by a limit 3 offset 1) as t1 order by a limit 2 offset 1", + "select * from (select * from t order by a limit 3) as t1 order by a limit 5", + "select b from (select b from t order by b limit 10 offset 10) as t1 order by b limit 10 offset 5", + "select b from (select b from t order by b limit 10 offset 2) as t1 order by b limit 3 offset 5", + "select a from (select a from t order by a limit 3 offset 5) as t1 order by a limit 3 offset 5", + "select a from (select a from t where b > 2 order by a, b limit 3 offset 1) as t1 order by a limit 2 offset 1", + "select * from (select * from t order by a limit 3) as t1 order by a, b limit 5" + ] } ] diff --git a/planner/cascades/testdata/integration_suite_out.json b/planner/cascades/testdata/integration_suite_out.json index de02b843ae95b..e25fdcf2ba1d7 100644 --- a/planner/cascades/testdata/integration_suite_out.json +++ b/planner/cascades/testdata/integration_suite_out.json @@ -595,5 +595,97 @@ "Result": null } ] + }, + { + "Name": "TestTopN", + "Cases": [ + { + "SQL": "select a from (select a from t where b > 2 order by a limit 3 offset 1) as t1 order by a limit 2 offset 1", + "Plan": [ + "Projection_25 2.00 root test.t.a", + "└─Limit_27 2.00 root offset:2, count:2", + " └─TableReader_35 4.00 root data:Limit_36", + " └─Limit_36 4.00 cop[tikv] offset:0, count:4", + " └─Selection_33 4.00 cop[tikv] gt(test.t.b, 2)", + " └─TableScan_34 4.00 cop[tikv] table:t, range:[-inf,+inf], keep order:true, stats:pseudo" + ], + "Result": [ + "3", + "4" + ] + }, + { + "SQL": "select * from (select * from t order by a limit 3) as t1 order by a limit 5", + "Plan": [ + "Limit_17 3.00 root offset:0, count:3", + "└─TableReader_23 3.00 root data:Limit_24", + " └─Limit_24 3.00 cop[tikv] offset:0, count:3", + " └─TableScan_22 3.00 cop[tikv] table:t, range:[-inf,+inf], keep order:true, stats:pseudo" + ], + "Result": [ + "1 11", + "2 22", + "3 33" + ] + }, + { + "SQL": "select b from (select b from t order by b limit 10 offset 10) as t1 order by b limit 10 offset 5", + "Plan": [ + "TopN_16 5.00 root test.t.b:asc, offset:15, count:5", + "└─TableReader_18 20.00 root data:TopN_19", + " └─TopN_19 20.00 cop[tikv] test.t.b:asc, offset:0, count:20", + " └─TableScan_21 10000.00 cop[tikv] table:t, range:[-inf,+inf], keep order:false, stats:pseudo" + ], + "Result": null + }, + { + "SQL": "select b from (select b from t order by b limit 10 offset 2) as t1 order by b limit 3 offset 5", + "Plan": [ + "TopN_16 3.00 root test.t.b:asc, offset:7, count:3", + "└─TableReader_18 10.00 root data:TopN_19", + " └─TopN_19 10.00 cop[tikv] test.t.b:asc, offset:0, count:10", + " └─TableScan_21 10000.00 cop[tikv] table:t, range:[-inf,+inf], keep order:false, stats:pseudo" + ], + "Result": null + }, + { + "SQL": "select a from (select a from t order by a limit 3 offset 5) as t1 order by a limit 3 offset 5", + "Plan": [ + "TableDual_14 0.00 root rows:0" + ], + "Result": null + }, + { + "SQL": "select a from (select a from t where b > 2 order by a, b limit 3 offset 1) as t1 order by a limit 2 offset 1", + "Plan": [ + "Projection_25 2.00 root test.t.a", + "└─TopN_26 2.00 root test.t.a:asc, test.t.b:asc, offset:2, count:2", + " └─TableReader_28 4.00 root data:TopN_29", + " └─TopN_29 4.00 cop[tikv] test.t.a:asc, test.t.b:asc, offset:0, count:4", + " └─Selection_31 8000.00 cop[tikv] gt(test.t.b, 2)", + " └─TableScan_32 10000.00 cop[tikv] table:t, range:[-inf,+inf], keep order:false, stats:pseudo" + ], + "Result": [ + "3", + "4" + ] + }, + { + "SQL": "select * from (select * from t order by a limit 3) as t1 order by a, b limit 5", + "Plan": [ + "Limit_14 3.00 root offset:0, count:5", + "└─Sort_26 3.00 root test.t.a:asc, test.t.b:asc", + " └─Limit_16 3.00 root offset:0, count:3", + " └─TableReader_22 3.00 root data:Limit_23", + " └─Limit_23 3.00 cop[tikv] offset:0, count:3", + " └─TableScan_21 3.00 cop[tikv] table:t, range:[-inf,+inf], keep order:true, stats:pseudo" + ], + "Result": [ + "1 11", + "2 22", + "3 33" + ] + } + ] } ] diff --git a/planner/cascades/testdata/transformation_rules_suite_in.json b/planner/cascades/testdata/transformation_rules_suite_in.json index a593df146a9f4..a7007f4c60f09 100644 --- a/planner/cascades/testdata/transformation_rules_suite_in.json +++ b/planner/cascades/testdata/transformation_rules_suite_in.json @@ -95,6 +95,23 @@ "select a from (select a from t limit 3 offset 5) t1 limit 3 offset 5" ] }, + { + "name": "TestMergeAdjacentTopN", + "cases": [ + "select b from (select b from t where c > 1 order by b limit 3) as t1 order by b limit 2", + "select a from (select a from t where b > 2 order by a limit 3 offset 1) as t1 order by a limit 2 offset 1", + "select * from (select * from t order by a limit 3) as t1 order by a limit 5", + "select b from (select b from t order by b limit 5) as t1 order by b limit 10", + "select b from (select b from t order by b limit 20) as t1 order by b limit 10", + "select b from (select b from t order by b limit 10) as t1 order by b limit 10", + "select b from (select b from t order by b limit 10 offset 10) as t1 order by b limit 10 offset 5", + "select b from (select b from t order by b limit 10 offset 2) as t1 order by b limit 3 offset 5", + "select b from (select b from t order by b limit 10 offset 5) as t1 order by b limit 5 offset 5", + "select a from (select a from t order by a limit 3 offset 5) as t1 order by a limit 3 offset 5", + "select b from (select b from t where c > 1 order by b, a limit 3) as t1 order by b limit 2", + "select a from (select a from t where b > 2 order by a, b limit 3 offset 1) as t1 order by a limit 2 offset 1" + ] + }, { "name": "TestTransformLimitToTableDual", "cases": [ diff --git a/planner/cascades/testdata/transformation_rules_suite_out.json b/planner/cascades/testdata/transformation_rules_suite_out.json index 6280fc2742e14..49863e4b328bc 100644 --- a/planner/cascades/testdata/transformation_rules_suite_out.json +++ b/planner/cascades/testdata/transformation_rules_suite_out.json @@ -1339,6 +1339,149 @@ } ] }, + { + "Name": "TestMergeAdjacentTopN", + "Cases": [ + { + "SQL": "select b from (select b from t where c > 1 order by b limit 3) as t1 order by b limit 2", + "Result": [ + "Group#0 Schema:[test.t.b]", + " Projection_11 input:[Group#1], test.t.b", + "Group#1 Schema:[test.t.b,test.t.c]", + " TopN_14 input:[Group#2], test.t.b:asc, offset:0, count:2", + "Group#2 Schema:[test.t.b,test.t.c]", + " Selection_2 input:[Group#3], gt(test.t.c, 1)", + "Group#3 Schema:[test.t.b,test.t.c]", + " TableScan_1 table:t" + ] + }, + { + "SQL": "select a from (select a from t where b > 2 order by a limit 3 offset 1) as t1 order by a limit 2 offset 1", + "Result": [ + "Group#0 Schema:[test.t.a]", + " Projection_11 input:[Group#1], test.t.a", + "Group#1 Schema:[test.t.a,test.t.b]", + " TopN_14 input:[Group#2], test.t.a:asc, offset:2, count:2", + "Group#2 Schema:[test.t.a,test.t.b]", + " Selection_2 input:[Group#3], gt(test.t.b, 2)", + "Group#3 Schema:[test.t.a,test.t.b]", + " TableScan_1 table:t" + ] + }, + { + "SQL": "select * from (select * from t order by a limit 3) as t1 order by a limit 5", + "Result": [ + "Group#0 Schema:[test.t.a,test.t.b,test.t.c,test.t.d,test.t.e,test.t.c_str,test.t.d_str,test.t.e_str,test.t.f,test.t.g,test.t.h,test.t.i_date]", + " Projection_10 input:[Group#1], test.t.a, test.t.b, test.t.c, test.t.d, test.t.e, test.t.c_str, test.t.d_str, test.t.e_str, test.t.f, test.t.g, test.t.h, test.t.i_date", + "Group#1 Schema:[test.t.a,test.t.b,test.t.c,test.t.d,test.t.e,test.t.c_str,test.t.d_str,test.t.e_str,test.t.f,test.t.g,test.t.h,test.t.i_date]", + " TopN_13 input:[Group#2], test.t.a:asc, offset:0, count:3", + "Group#2 Schema:[test.t.a,test.t.b,test.t.c,test.t.d,test.t.e,test.t.c_str,test.t.d_str,test.t.e_str,test.t.f,test.t.g,test.t.h,test.t.i_date]", + " TableScan_1 table:t" + ] + }, + { + "SQL": "select b from (select b from t order by b limit 5) as t1 order by b limit 10", + "Result": [ + "Group#0 Schema:[test.t.b]", + " Projection_10 input:[Group#1], test.t.b", + "Group#1 Schema:[test.t.b]", + " TopN_13 input:[Group#2], test.t.b:asc, offset:0, count:5", + "Group#2 Schema:[test.t.b]", + " TableScan_1 table:t" + ] + }, + { + "SQL": "select b from (select b from t order by b limit 20) as t1 order by b limit 10", + "Result": [ + "Group#0 Schema:[test.t.b]", + " Projection_10 input:[Group#1], test.t.b", + "Group#1 Schema:[test.t.b]", + " TopN_13 input:[Group#2], test.t.b:asc, offset:0, count:10", + "Group#2 Schema:[test.t.b]", + " TableScan_1 table:t" + ] + }, + { + "SQL": "select b from (select b from t order by b limit 10) as t1 order by b limit 10", + "Result": [ + "Group#0 Schema:[test.t.b]", + " Projection_10 input:[Group#1], test.t.b", + "Group#1 Schema:[test.t.b]", + " TopN_13 input:[Group#2], test.t.b:asc, offset:0, count:10", + "Group#2 Schema:[test.t.b]", + " TableScan_1 table:t" + ] + }, + { + "SQL": "select b from (select b from t order by b limit 10 offset 10) as t1 order by b limit 10 offset 5", + "Result": [ + "Group#0 Schema:[test.t.b]", + " Projection_10 input:[Group#1], test.t.b", + "Group#1 Schema:[test.t.b]", + " TopN_13 input:[Group#2], test.t.b:asc, offset:15, count:5", + "Group#2 Schema:[test.t.b]", + " TableScan_1 table:t" + ] + }, + { + "SQL": "select b from (select b from t order by b limit 10 offset 2) as t1 order by b limit 3 offset 5", + "Result": [ + "Group#0 Schema:[test.t.b]", + " Projection_10 input:[Group#1], test.t.b", + "Group#1 Schema:[test.t.b]", + " TopN_13 input:[Group#2], test.t.b:asc, offset:7, count:3", + "Group#2 Schema:[test.t.b]", + " TableScan_1 table:t" + ] + }, + { + "SQL": "select b from (select b from t order by b limit 10 offset 5) as t1 order by b limit 5 offset 5", + "Result": [ + "Group#0 Schema:[test.t.b]", + " Projection_10 input:[Group#1], test.t.b", + "Group#1 Schema:[test.t.b]", + " TopN_13 input:[Group#2], test.t.b:asc, offset:10, count:5", + "Group#2 Schema:[test.t.b]", + " TableScan_1 table:t" + ] + }, + { + "SQL": "select a from (select a from t order by a limit 3 offset 5) as t1 order by a limit 3 offset 5", + "Result": [ + "Group#0 Schema:[test.t.a]", + " Projection_10 input:[Group#1], test.t.a", + "Group#1 Schema:[test.t.a]", + " TableDual_13 rowcount:0" + ] + }, + { + "SQL": "select b from (select b from t where c > 1 order by b, a limit 3) as t1 order by b limit 2", + "Result": [ + "Group#0 Schema:[test.t.b]", + " Projection_13 input:[Group#1], test.t.b", + "Group#1 Schema:[test.t.a,test.t.b,test.t.c]", + " TopN_16 input:[Group#2], test.t.b:asc, test.t.a:asc, offset:0, count:2", + "Group#2 Schema:[test.t.a,test.t.b,test.t.c]", + " Selection_2 input:[Group#3], gt(test.t.c, 1)", + "Group#3 Schema:[test.t.a,test.t.b,test.t.c]", + " TableScan_1 table:t" + ] + }, + { + "SQL": "select a from (select a from t where b > 2 order by a, b limit 3 offset 1) as t1 order by a limit 2 offset 1", + "Result": [ + "Group#0 Schema:[test.t.a]", + " Projection_13 input:[Group#1], test.t.a", + "Group#1 Schema:[test.t.a,test.t.b]", + " TopN_16 input:[Group#2], test.t.a:asc, test.t.b:asc, offset:2, count:2", + "Group#2 Schema:[test.t.a,test.t.b]", + " Selection_2 input:[Group#3], gt(test.t.b, 2)", + "Group#3 Schema:[test.t.a,test.t.b]", + " TableScan_1 table:t" + ] + } + ] + }, { "Name": "TestTransformLimitToTableDual", "Cases": [ diff --git a/planner/cascades/transformation_rules.go b/planner/cascades/transformation_rules.go index 5b044b48cb6dd..b04ccb8c9e477 100644 --- a/planner/cascades/transformation_rules.go +++ b/planner/cascades/transformation_rules.go @@ -85,6 +85,7 @@ var defaultTransformationMap = map[memo.Operand][]Transformation{ NewRulePushTopNDownOuterJoin(), NewRulePushTopNDownUnionAll(), NewRulePushTopNDownTiKVSingleGather(), + NewRuleMergeAdjacentTopN(), }, } @@ -1283,6 +1284,66 @@ func (r *PushTopNDownTiKVSingleGather) OnTransform(old *memo.ExprIter) (newExprs return []*memo.GroupExpr{finalTopNExpr}, true, false, nil } +// MergeAdjacentTopN merge adjacent TopN. +type MergeAdjacentTopN struct { + baseRule +} + +// NewRuleMergeAdjacentTopN creates a new Transformation MergeAdjacentTopN. +// The pattern of this rule is `TopN->TopN->X`. +func NewRuleMergeAdjacentTopN() Transformation { + rule := &MergeAdjacentTopN{} + rule.pattern = memo.BuildPattern( + memo.OperandTopN, + memo.EngineAll, + memo.NewPattern(memo.OperandTopN, memo.EngineAll), + ) + return rule +} + +// Match implements Transformation interface. +func (r *MergeAdjacentTopN) Match(expr *memo.ExprIter) bool { + topN := expr.GetExpr().ExprNode.(*plannercore.LogicalTopN) + child := expr.Children[0].GetExpr().ExprNode.(*plannercore.LogicalTopN) + + // We can use this rule when the sort columns of parent TopN is a prefix of child TopN. + if len(child.ByItems) < len(topN.ByItems) { + return false + } + for i := 0; i < len(topN.ByItems); i++ { + if !topN.ByItems[i].Equal(topN.SCtx(), child.ByItems[i]) { + return false + } + } + return true +} + +// OnTransform implements Transformation interface. +// This rule tries to merge adjacent TopN. +func (r *MergeAdjacentTopN) OnTransform(old *memo.ExprIter) (newExprs []*memo.GroupExpr, eraseOld bool, eraseAll bool, err error) { + topN := old.GetExpr().ExprNode.(*plannercore.LogicalTopN) + child := old.Children[0].GetExpr().ExprNode.(*plannercore.LogicalTopN) + childGroups := old.Children[0].GetExpr().Children + + if child.Count <= topN.Offset { + tableDual := plannercore.LogicalTableDual{RowCount: 0}.Init(child.SCtx(), child.SelectBlockOffset()) + tableDual.SetSchema(old.GetExpr().Schema()) + tableDualExpr := memo.NewGroupExpr(tableDual) + return []*memo.GroupExpr{tableDualExpr}, true, true, nil + } + + offset := child.Offset + topN.Offset + count := uint64(math.Min(float64(child.Count-topN.Offset), float64(topN.Count))) + newTopN := plannercore.LogicalTopN{ + Count: count, + Offset: offset, + ByItems: child.ByItems, + }.Init(child.SCtx(), child.SelectBlockOffset()) + newTopNExpr := memo.NewGroupExpr(newTopN) + newTopNExpr.SetChildren(childGroups...) + return []*memo.GroupExpr{newTopNExpr}, true, false, nil +} + // MergeAggregationProjection merges the Projection below an Aggregation as a new Aggregation. // The Projection may be regenerated in the ImplementationPhase. But this rule allows the // Aggregation to match other rules, such as MergeAdjacentAggregation. diff --git a/planner/cascades/transformation_rules_test.go b/planner/cascades/transformation_rules_test.go index 3bff81b6400dd..aa821c0158cf9 100644 --- a/planner/cascades/transformation_rules_test.go +++ b/planner/cascades/transformation_rules_test.go @@ -228,6 +228,31 @@ func (s *testTransformationRuleSuite) TestMergeAggregationProjection(c *C) { testGroupToString(input, output, s, c) } +func (s *testTransformationRuleSuite) TestMergeAdjacentTopN(c *C) { + s.optimizer.ResetTransformationRules(map[memo.Operand][]Transformation{ + memo.OperandLimit: { + NewRuleTransformLimitToTopN(), + }, + memo.OperandTopN: { + NewRulePushTopNDownProjection(), + NewRuleMergeAdjacentTopN(), + }, + memo.OperandProjection: { + NewRuleMergeAdjacentProjection(), + }, + }) + defer func() { + s.optimizer.ResetTransformationRules(defaultTransformationMap) + }() + var input []string + var output []struct { + SQL string + Result []string + } + s.testData.GetTestCases(c, &input, &output) + testGroupToString(input, output, s, c) +} + func (s *testTransformationRuleSuite) TestMergeAdjacentLimit(c *C) { s.optimizer.ResetTransformationRules(map[memo.Operand][]Transformation{ memo.OperandLimit: { diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index bda7da337dd4a..abf96d5f8fe4b 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -1148,6 +1148,11 @@ func (by *ByItems) Clone() *ByItems { return &ByItems{Expr: by.Expr.Clone(), Desc: by.Desc} } +// Equal checks whether two ByItems are equal. +func (by *ByItems) Equal(ctx sessionctx.Context, other *ByItems) bool { + return by.Expr.Equal(ctx, other.Expr) && by.Desc == other.Desc +} + // itemTransformer transforms ParamMarkerExpr to PositionExpr in the context of ByItem type itemTransformer struct { }