From f76abb3b6711df0cf137e5b5e5fefdfb8d7c0960 Mon Sep 17 00:00:00 2001 From: AI-Login-is-already-taken <956308585@qq.com> Date: Mon, 23 Nov 2020 23:39:43 +0800 Subject: [PATCH 01/18] fix statement-level optimize hint will be ignored when `tryFastPlan` works and enable memory tracker in point_get/batch_point_get --- executor/batch_point_get.go | 10 ++++++++++ executor/executor.go | 8 ++++++++ executor/point_get.go | 10 ++++++++++ planner/optimize.go | 28 ++++++++++++++-------------- session/session_test.go | 18 ++++++++++++++++++ 5 files changed, 60 insertions(+), 14 deletions(-) diff --git a/executor/batch_point_get.go b/executor/batch_point_get.go index 826a88d3aacdf..175c17d2be78b 100644 --- a/executor/batch_point_get.go +++ b/executor/batch_point_get.go @@ -16,6 +16,7 @@ package executor import ( "context" "fmt" + "github.com/pingcap/tidb/util/memory" "sort" "sync/atomic" @@ -68,6 +69,8 @@ type BatchPointGetExec struct { snapshot kv.Snapshot stats *runtimeStatsWithSnapshot + + memTracker *memory.Tracker } // buildVirtualColumnInfo saves virtual column indices and sort them in definition order @@ -119,6 +122,13 @@ func (e *BatchPointGetExec) Open(context.Context) error { } e.snapshot = snapshot e.batchGetter = batchGetter + + if e.memTracker == nil { + e.memTracker = memory.NewTracker(e.id, -1) + e.memTracker.AttachTo(e.ctx.GetSessionVars().StmtCtx.MemTracker) + result := newFirstChunk(e) + e.memTracker.Consume(result.MemoryUsage()) + } return nil } diff --git a/executor/executor.go b/executor/executor.go index fc56e123bfabd..1c98e9b1fcf9b 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -1210,6 +1210,14 @@ type SelectionExec struct { // Open implements the Executor Open interface. func (e *SelectionExec) Open(ctx context.Context) error { + if len(e.baseExecutor.children) > 0 { + child := e.baseExecutor.children[0] + if batchPointGet, ok := child.(*BatchPointGetExec); ok { + batchPointGet.memTracker = memory.NewTracker(batchPointGet.id, -1) + } else if pointGet, ok := child.(*PointGetExecutor); ok { + pointGet.memTracker = memory.NewTracker(pointGet.id, -1) + } + } if err := e.baseExecutor.Open(ctx); err != nil { return err } diff --git a/executor/point_get.go b/executor/point_get.go index 0dd73eec70b33..6c7dbbd736e91 100644 --- a/executor/point_get.go +++ b/executor/point_get.go @@ -15,6 +15,7 @@ package executor import ( "context" + "github.com/pingcap/tidb/util/memory" "github.com/pingcap/errors" "github.com/pingcap/failpoint" @@ -87,6 +88,8 @@ type PointGetExecutor struct { virtualColumnRetFieldTypes []*types.FieldType stats *runtimeStatsWithSnapshot + + memTracker *memory.Tracker } // Init set fields needed for PointGetExecutor reuse, this does NOT change baseExecutor field @@ -146,6 +149,13 @@ func (e *PointGetExecutor) Open(context.Context) error { e.snapshot.SetOption(kv.ReplicaRead, kv.ReplicaReadFollower) } e.snapshot.SetOption(kv.TaskID, e.ctx.GetSessionVars().StmtCtx.TaskID) + + if e.memTracker == nil { + e.memTracker = memory.NewTracker(e.id, -1) + e.memTracker.AttachTo(e.ctx.GetSessionVars().StmtCtx.MemTracker) + result := newFirstChunk(e) + e.memTracker.Consume(result.MemoryUsage()) + } return nil } diff --git a/planner/optimize.go b/planner/optimize.go index 26e9e6088bd4f..03449610bd5b7 100644 --- a/planner/optimize.go +++ b/planner/optimize.go @@ -87,6 +87,20 @@ func Optimize(ctx context.Context, sctx sessionctx.Context, node ast.Node, is in }() } + tableHints := hint.ExtractTableHintsFromStmtNode(node, sctx) + stmtHints, warns := handleStmtHints(tableHints) + sessVars.StmtCtx.StmtHints = stmtHints + for _, warn := range warns { + sctx.GetSessionVars().StmtCtx.AppendWarning(warn) + } + warns = warns[:0] + for name, val := range stmtHints.SetVars { + err := variable.SetStmtVar(sessVars, name, val) + if err != nil { + sctx.GetSessionVars().StmtCtx.AppendWarning(err) + } + } + if _, isolationReadContainTiKV := sessVars.IsolationReadEngines[kv.TiKV]; isolationReadContainTiKV { var fp plannercore.Plan if fpv, ok := sctx.Value(plannercore.PointPlanKey).(plannercore.PointPlanVal); ok { @@ -105,20 +119,6 @@ func Optimize(ctx context.Context, sctx sessionctx.Context, node ast.Node, is in sctx.PrepareTSFuture(ctx) - tableHints := hint.ExtractTableHintsFromStmtNode(node, sctx) - stmtHints, warns := handleStmtHints(tableHints) - sessVars.StmtCtx.StmtHints = stmtHints - for _, warn := range warns { - sctx.GetSessionVars().StmtCtx.AppendWarning(warn) - } - warns = warns[:0] - for name, val := range stmtHints.SetVars { - err := variable.SetStmtVar(sessVars, name, val) - if err != nil { - sctx.GetSessionVars().StmtCtx.AppendWarning(err) - } - } - bestPlan, names, _, err := optimize(ctx, sctx, node, is) if err != nil { return nil, nil, err diff --git a/session/session_test.go b/session/session_test.go index e8dd43126a67f..18512a1280c5c 100644 --- a/session/session_test.go +++ b/session/session_test.go @@ -3139,6 +3139,24 @@ func (s *testSessionSuite2) TestStmtHints(c *C) { tk.MustExec("select /*+ READ_CONSISTENT_REPLICA(), READ_CONSISTENT_REPLICA() */ 1;") c.Assert(tk.Se.GetSessionVars().StmtCtx.GetWarnings(), HasLen, 1) c.Assert(tk.Se.GetSessionVars().GetReplicaRead(), Equals, kv.ReplicaReadFollower) + + // Test MEMORY_QUOTA hint in POINT_GET/BATCH_POINT_GET + tk.MustExec("use test") + tk.MustExec("drop table if exists t1, t1") + tk.MustExec("create table t1(a int);") + tk.MustExec("insert /*+ MEMORY_QUOTA(1 GB) */ into t1 values (1);") + tk.MustExec("select /*+ MEMORY_QUOTA(0 GB) */ * from t1 where a = 1;") + val = int64(0) + c.Assert(tk.Se.GetSessionVars().StmtCtx.MemTracker.CheckBytesLimit(val), IsTrue) + c.Assert(tk.Se.GetSessionVars().StmtCtx.GetWarnings()[0].Err.Error(), Equals, "Setting the MEMORY_QUOTA to 0 means no memory limit") + tk.MustExec("set tidb_mem_quota_query=1;") + _, err = tk.Exec("select * from t1 where a = 1;") + c.Assert(err.Error(), Matches, "Out Of Memory Quota!.*") + tk.MustQuery("select /*+ MEMORY_QUOTA(1 GB) */ * from t1 where a = 1;").Check(testkit.Rows("1")) + _, err = tk.Exec("update t1 set a = a + 10 where a in (1,2,3);") + c.Assert(err.Error(), Matches, "Out Of Memory Quota!.*") + _, err = tk.Exec("update /*+ memory_quota(1 MB) */ t1 set a = a + 10 where a in (1,2,3);") + c.Check(err, IsNil) } func (s *testSessionSuite3) TestPessimisticLockOnPartition(c *C) { From 5c81c3fde7e09a40c56d7ada66948e51981f974d Mon Sep 17 00:00:00 2001 From: AI-Login-is-already-taken <956308585@qq.com> Date: Tue, 24 Nov 2020 00:00:33 +0800 Subject: [PATCH 02/18] fix import,test sql --- executor/batch_point_get.go | 2 +- session/session_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/executor/batch_point_get.go b/executor/batch_point_get.go index 175c17d2be78b..591a663de4bb5 100644 --- a/executor/batch_point_get.go +++ b/executor/batch_point_get.go @@ -16,7 +16,6 @@ package executor import ( "context" "fmt" - "github.com/pingcap/tidb/util/memory" "sort" "sync/atomic" @@ -33,6 +32,7 @@ import ( "github.com/pingcap/tidb/util/codec" "github.com/pingcap/tidb/util/hack" "github.com/pingcap/tidb/util/math" + "github.com/pingcap/tidb/util/memory" "github.com/pingcap/tidb/util/rowcodec" ) diff --git a/session/session_test.go b/session/session_test.go index 18512a1280c5c..1a650a3175d80 100644 --- a/session/session_test.go +++ b/session/session_test.go @@ -3142,7 +3142,7 @@ func (s *testSessionSuite2) TestStmtHints(c *C) { // Test MEMORY_QUOTA hint in POINT_GET/BATCH_POINT_GET tk.MustExec("use test") - tk.MustExec("drop table if exists t1, t1") + tk.MustExec("drop table if exists t1, t1;") tk.MustExec("create table t1(a int);") tk.MustExec("insert /*+ MEMORY_QUOTA(1 GB) */ into t1 values (1);") tk.MustExec("select /*+ MEMORY_QUOTA(0 GB) */ * from t1 where a = 1;") From 83f42480206459543bda7a9ac82425de1ef2f9a3 Mon Sep 17 00:00:00 2001 From: AI-Login-is-already-taken <956308585@qq.com> Date: Tue, 24 Nov 2020 00:35:17 +0800 Subject: [PATCH 03/18] fix error --- executor/point_get.go | 2 +- session/session_test.go | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/executor/point_get.go b/executor/point_get.go index 6c7dbbd736e91..93491936a7c07 100644 --- a/executor/point_get.go +++ b/executor/point_get.go @@ -15,7 +15,6 @@ package executor import ( "context" - "github.com/pingcap/tidb/util/memory" "github.com/pingcap/errors" "github.com/pingcap/failpoint" @@ -33,6 +32,7 @@ import ( "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/codec" "github.com/pingcap/tidb/util/execdetails" + "github.com/pingcap/tidb/util/memory" "github.com/pingcap/tidb/util/rowcodec" ) diff --git a/session/session_test.go b/session/session_test.go index 1a650a3175d80..c5b6ae1c9b741 100644 --- a/session/session_test.go +++ b/session/session_test.go @@ -3142,7 +3142,6 @@ func (s *testSessionSuite2) TestStmtHints(c *C) { // Test MEMORY_QUOTA hint in POINT_GET/BATCH_POINT_GET tk.MustExec("use test") - tk.MustExec("drop table if exists t1, t1;") tk.MustExec("create table t1(a int);") tk.MustExec("insert /*+ MEMORY_QUOTA(1 GB) */ into t1 values (1);") tk.MustExec("select /*+ MEMORY_QUOTA(0 GB) */ * from t1 where a = 1;") From 7f58a62e299c84950aa516d07dc6d618a7e38689 Mon Sep 17 00:00:00 2001 From: AI-Login-is-already-taken <956308585@qq.com> Date: Tue, 24 Nov 2020 00:41:29 +0800 Subject: [PATCH 04/18] fix unit test --- session/session_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/session/session_test.go b/session/session_test.go index c5b6ae1c9b741..2acff2e779a57 100644 --- a/session/session_test.go +++ b/session/session_test.go @@ -3141,8 +3141,6 @@ func (s *testSessionSuite2) TestStmtHints(c *C) { c.Assert(tk.Se.GetSessionVars().GetReplicaRead(), Equals, kv.ReplicaReadFollower) // Test MEMORY_QUOTA hint in POINT_GET/BATCH_POINT_GET - tk.MustExec("use test") - tk.MustExec("create table t1(a int);") tk.MustExec("insert /*+ MEMORY_QUOTA(1 GB) */ into t1 values (1);") tk.MustExec("select /*+ MEMORY_QUOTA(0 GB) */ * from t1 where a = 1;") val = int64(0) From e4aa904f57958b8704ce295c2ea7d0211563e1e7 Mon Sep 17 00:00:00 2001 From: AI-Login-is-already-taken <956308585@qq.com> Date: Tue, 24 Nov 2020 01:19:08 +0800 Subject: [PATCH 05/18] fix unit test --- session/session_test.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/session/session_test.go b/session/session_test.go index 2acff2e779a57..4581ef46a23fb 100644 --- a/session/session_test.go +++ b/session/session_test.go @@ -3139,15 +3139,22 @@ func (s *testSessionSuite2) TestStmtHints(c *C) { tk.MustExec("select /*+ READ_CONSISTENT_REPLICA(), READ_CONSISTENT_REPLICA() */ 1;") c.Assert(tk.Se.GetSessionVars().StmtCtx.GetWarnings(), HasLen, 1) c.Assert(tk.Se.GetSessionVars().GetReplicaRead(), Equals, kv.ReplicaReadFollower) +} + +// Test MEMORY_QUOTA hint in POINT_GET/BATCH_POINT_GET +func (s *testSessionSuite2) TestPointGetStmtHints(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t1;") + tk.MustExec("create table t1(a int);") - // Test MEMORY_QUOTA hint in POINT_GET/BATCH_POINT_GET tk.MustExec("insert /*+ MEMORY_QUOTA(1 GB) */ into t1 values (1);") tk.MustExec("select /*+ MEMORY_QUOTA(0 GB) */ * from t1 where a = 1;") - val = int64(0) + val := int64(0) c.Assert(tk.Se.GetSessionVars().StmtCtx.MemTracker.CheckBytesLimit(val), IsTrue) c.Assert(tk.Se.GetSessionVars().StmtCtx.GetWarnings()[0].Err.Error(), Equals, "Setting the MEMORY_QUOTA to 0 means no memory limit") tk.MustExec("set tidb_mem_quota_query=1;") - _, err = tk.Exec("select * from t1 where a = 1;") + _, err := tk.Exec("select * from t1 where a = 1;") c.Assert(err.Error(), Matches, "Out Of Memory Quota!.*") tk.MustQuery("select /*+ MEMORY_QUOTA(1 GB) */ * from t1 where a = 1;").Check(testkit.Rows("1")) _, err = tk.Exec("update t1 set a = a + 10 where a in (1,2,3);") From 291e2c657cc25fb4f8b5b22ad210544d1d6cd288 Mon Sep 17 00:00:00 2001 From: AI-Login-is-already-taken <956308585@qq.com> Date: Tue, 24 Nov 2020 01:36:51 +0800 Subject: [PATCH 06/18] fix unit test --- session/session_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/session/session_test.go b/session/session_test.go index 4581ef46a23fb..a867895a2158d 100644 --- a/session/session_test.go +++ b/session/session_test.go @@ -3147,7 +3147,10 @@ func (s *testSessionSuite2) TestPointGetStmtHints(c *C) { tk.MustExec("use test") tk.MustExec("drop table if exists t1;") tk.MustExec("create table t1(a int);") - + defer config.RestoreFunc()() + config.UpdateGlobal(func(conf *config.Config) { + conf.OOMAction = config.OOMActionCancel + }) tk.MustExec("insert /*+ MEMORY_QUOTA(1 GB) */ into t1 values (1);") tk.MustExec("select /*+ MEMORY_QUOTA(0 GB) */ * from t1 where a = 1;") val := int64(0) From c9f3966c49296728861a5be6c879e665e7a14b8b Mon Sep 17 00:00:00 2001 From: AI-Login-is-already-taken <956308585@qq.com> Date: Tue, 24 Nov 2020 01:44:13 +0800 Subject: [PATCH 07/18] fix unit test --- session/session_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/session/session_test.go b/session/session_test.go index a867895a2158d..b85d404159d1f 100644 --- a/session/session_test.go +++ b/session/session_test.go @@ -3157,12 +3157,12 @@ func (s *testSessionSuite2) TestPointGetStmtHints(c *C) { c.Assert(tk.Se.GetSessionVars().StmtCtx.MemTracker.CheckBytesLimit(val), IsTrue) c.Assert(tk.Se.GetSessionVars().StmtCtx.GetWarnings()[0].Err.Error(), Equals, "Setting the MEMORY_QUOTA to 0 means no memory limit") tk.MustExec("set tidb_mem_quota_query=1;") - _, err := tk.Exec("select * from t1 where a = 1;") + err := tk.QueryToErr("select * from t1 where a = 1;") c.Assert(err.Error(), Matches, "Out Of Memory Quota!.*") tk.MustQuery("select /*+ MEMORY_QUOTA(1 GB) */ * from t1 where a = 1;").Check(testkit.Rows("1")) - _, err = tk.Exec("update t1 set a = a + 10 where a in (1,2,3);") + err = tk.ExecToErr("update t1 set a = a + 10 where a in (1,2,3);") c.Assert(err.Error(), Matches, "Out Of Memory Quota!.*") - _, err = tk.Exec("update /*+ memory_quota(1 MB) */ t1 set a = a + 10 where a in (1,2,3);") + err = tk.ExecToErr("update /*+ memory_quota(1 MB) */ t1 set a = a + 10 where a in (1,2,3);") c.Check(err, IsNil) } From 87d0476af71c89fd5a09474e6d88342600f3aadd Mon Sep 17 00:00:00 2001 From: AI-Login-is-already-taken <956308585@qq.com> Date: Fri, 11 Dec 2020 10:10:13 +0800 Subject: [PATCH 08/18] fix test --- session/session_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/session/session_test.go b/session/session_test.go index b85d404159d1f..49bab6202aede 100644 --- a/session/session_test.go +++ b/session/session_test.go @@ -3146,7 +3146,7 @@ func (s *testSessionSuite2) TestPointGetStmtHints(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec("drop table if exists t1;") - tk.MustExec("create table t1(a int);") + tk.MustExec("create table t1(a int primary key);") defer config.RestoreFunc()() config.UpdateGlobal(func(conf *config.Config) { conf.OOMAction = config.OOMActionCancel From 4ff817217017c7cf4e6151b326090b3b24184090 Mon Sep 17 00:00:00 2001 From: AI-Login-is-already-taken <956308585@qq.com> Date: Sun, 13 Dec 2020 23:09:30 +0800 Subject: [PATCH 09/18] fix memory track --- executor/adapter.go | 7 +++++++ executor/batch_point_get.go | 9 ++++++--- executor/builder.go | 3 +++ executor/executor.go | 8 -------- executor/point_get.go | 12 +++++++++--- planner/core/point_get_plan.go | 2 ++ session/session_test.go | 1 + 7 files changed, 28 insertions(+), 14 deletions(-) diff --git a/executor/adapter.go b/executor/adapter.go index d52a1a442391e..a9d20b3d115a6 100644 --- a/executor/adapter.go +++ b/executor/adapter.go @@ -743,6 +743,13 @@ func (a *ExecStmt) buildExecutor() (Executor, error) { } b := newExecutorBuilder(ctx, a.InfoSchema) + + if pg, ok := a.Plan.(*plannercore.PointGetPlan); ok { + pg.TrackMem = true + } + if bpg, ok := a.Plan.(*plannercore.BatchPointGetPlan); ok { + bpg.TrackMem = true + } e := b.build(a.Plan) if b.err != nil { return nil, errors.Trace(b.err) diff --git a/executor/batch_point_get.go b/executor/batch_point_get.go index 591a663de4bb5..6a1746dcaaa9a 100644 --- a/executor/batch_point_get.go +++ b/executor/batch_point_get.go @@ -70,6 +70,8 @@ type BatchPointGetExec struct { snapshot kv.Snapshot stats *runtimeStatsWithSnapshot + // control if track mem + trackMem bool memTracker *memory.Tracker } @@ -123,11 +125,9 @@ func (e *BatchPointGetExec) Open(context.Context) error { e.snapshot = snapshot e.batchGetter = batchGetter - if e.memTracker == nil { + if e.trackMem { e.memTracker = memory.NewTracker(e.id, -1) e.memTracker.AttachTo(e.ctx.GetSessionVars().StmtCtx.MemTracker) - result := newFirstChunk(e) - e.memTracker.Consume(result.MemoryUsage()) } return nil } @@ -159,6 +159,9 @@ func (e *BatchPointGetExec) Next(ctx context.Context, req *chunk.Chunk) error { } for !req.IsFull() && e.index < len(e.values) { handle, val := e.handles[e.index], e.values[e.index] + if e.memTracker != nil { + e.memTracker.Consume(int64(len(val))) + } err := DecodeRowValToChunk(e.base().ctx, e.schema, e.tblInfo, handle, val, req, e.rowDecoder) if err != nil { return err diff --git a/executor/builder.go b/executor/builder.go index ad128c559b484..96ef80628d129 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -3812,6 +3812,9 @@ func (b *executorBuilder) buildBatchPointGet(plan *plannercore.BatchPointGetPlan if e.lock { b.hasLock = true } + if plan.TrackMem { + e.trackMem = true + } var capacity int if plan.IndexInfo != nil && !isCommonHandleRead(plan.TblInfo, plan.IndexInfo) { e.idxVals = plan.IndexValues diff --git a/executor/executor.go b/executor/executor.go index 1c98e9b1fcf9b..fc56e123bfabd 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -1210,14 +1210,6 @@ type SelectionExec struct { // Open implements the Executor Open interface. func (e *SelectionExec) Open(ctx context.Context) error { - if len(e.baseExecutor.children) > 0 { - child := e.baseExecutor.children[0] - if batchPointGet, ok := child.(*BatchPointGetExec); ok { - batchPointGet.memTracker = memory.NewTracker(batchPointGet.id, -1) - } else if pointGet, ok := child.(*PointGetExecutor); ok { - pointGet.memTracker = memory.NewTracker(pointGet.id, -1) - } - } if err := e.baseExecutor.Open(ctx); err != nil { return err } diff --git a/executor/point_get.go b/executor/point_get.go index 93491936a7c07..ee2f09e0c419e 100644 --- a/executor/point_get.go +++ b/executor/point_get.go @@ -56,6 +56,9 @@ func (b *executorBuilder) buildPointGet(p *plannercore.PointGetPlan) Executor { if p.Lock { b.hasLock = true } + if p.TrackMem { + e.trackMem = true + } e.Init(p, startTS) return e } @@ -89,6 +92,8 @@ type PointGetExecutor struct { stats *runtimeStatsWithSnapshot + // control if track mem + trackMem bool memTracker *memory.Tracker } @@ -150,11 +155,9 @@ func (e *PointGetExecutor) Open(context.Context) error { } e.snapshot.SetOption(kv.TaskID, e.ctx.GetSessionVars().StmtCtx.TaskID) - if e.memTracker == nil { + if e.trackMem { e.memTracker = memory.NewTracker(e.id, -1) e.memTracker.AttachTo(e.ctx.GetSessionVars().StmtCtx.MemTracker) - result := newFirstChunk(e) - e.memTracker.Consume(result.MemoryUsage()) } return nil } @@ -257,6 +260,9 @@ func (e *PointGetExecutor) Next(ctx context.Context, req *chunk.Chunk) error { } return nil } + if e.memTracker != nil { + e.memTracker.Consume(int64(len(val))) + } err = DecodeRowValToChunk(e.base().ctx, e.schema, e.tblInfo, e.handle, val, req, e.rowDecoder) if err != nil { return err diff --git a/planner/core/point_get_plan.go b/planner/core/point_get_plan.go index c6cf3b25260a9..9c2c5b4e7e8cc 100644 --- a/planner/core/point_get_plan.go +++ b/planner/core/point_get_plan.go @@ -66,6 +66,7 @@ type PointGetPlan struct { LockWaitTime int64 partitionColumnPos int Columns []*model.ColumnInfo + TrackMem bool } type nameValuePair struct { @@ -257,6 +258,7 @@ type BatchPointGetPlan struct { Lock bool LockWaitTime int64 Columns []*model.ColumnInfo + TrackMem bool } // Clone implements PhysicalPlan interface. diff --git a/session/session_test.go b/session/session_test.go index 49bab6202aede..e73ef81ad9b9b 100644 --- a/session/session_test.go +++ b/session/session_test.go @@ -3158,6 +3158,7 @@ func (s *testSessionSuite2) TestPointGetStmtHints(c *C) { c.Assert(tk.Se.GetSessionVars().StmtCtx.GetWarnings()[0].Err.Error(), Equals, "Setting the MEMORY_QUOTA to 0 means no memory limit") tk.MustExec("set tidb_mem_quota_query=1;") err := tk.QueryToErr("select * from t1 where a = 1;") + c.Assert(err, NotNil) c.Assert(err.Error(), Matches, "Out Of Memory Quota!.*") tk.MustQuery("select /*+ MEMORY_QUOTA(1 GB) */ * from t1 where a = 1;").Check(testkit.Rows("1")) err = tk.ExecToErr("update t1 set a = a + 10 where a in (1,2,3);") From 6ada44822f722fb9fef4bb0cf51f2bda74c20840 Mon Sep 17 00:00:00 2001 From: AI-Login-is-already-taken <956308585@qq.com> Date: Sun, 13 Dec 2020 23:30:43 +0800 Subject: [PATCH 10/18] fix test,revert changes about memory hit --- planner/optimize.go | 29 ++++++++++++++--------------- session/session_test.go | 13 ++++++------- 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/planner/optimize.go b/planner/optimize.go index 03449610bd5b7..627c55eb9e153 100644 --- a/planner/optimize.go +++ b/planner/optimize.go @@ -86,21 +86,6 @@ func Optimize(ctx context.Context, sctx sessionctx.Context, node ast.Node, is in sessVars.IsolationReadEngines[kv.TiFlash] = struct{}{} }() } - - tableHints := hint.ExtractTableHintsFromStmtNode(node, sctx) - stmtHints, warns := handleStmtHints(tableHints) - sessVars.StmtCtx.StmtHints = stmtHints - for _, warn := range warns { - sctx.GetSessionVars().StmtCtx.AppendWarning(warn) - } - warns = warns[:0] - for name, val := range stmtHints.SetVars { - err := variable.SetStmtVar(sessVars, name, val) - if err != nil { - sctx.GetSessionVars().StmtCtx.AppendWarning(err) - } - } - if _, isolationReadContainTiKV := sessVars.IsolationReadEngines[kv.TiKV]; isolationReadContainTiKV { var fp plannercore.Plan if fpv, ok := sctx.Value(plannercore.PointPlanKey).(plannercore.PointPlanVal); ok { @@ -119,6 +104,20 @@ func Optimize(ctx context.Context, sctx sessionctx.Context, node ast.Node, is in sctx.PrepareTSFuture(ctx) + tableHints := hint.ExtractTableHintsFromStmtNode(node, sctx) + stmtHints, warns := handleStmtHints(tableHints) + sessVars.StmtCtx.StmtHints = stmtHints + for _, warn := range warns { + sctx.GetSessionVars().StmtCtx.AppendWarning(warn) + } + warns = warns[:0] + for name, val := range stmtHints.SetVars { + err := variable.SetStmtVar(sessVars, name, val) + if err != nil { + sctx.GetSessionVars().StmtCtx.AppendWarning(err) + } + } + bestPlan, names, _, err := optimize(ctx, sctx, node, is) if err != nil { return nil, nil, err diff --git a/session/session_test.go b/session/session_test.go index e73ef81ad9b9b..bff43874c67d9 100644 --- a/session/session_test.go +++ b/session/session_test.go @@ -3151,19 +3151,18 @@ func (s *testSessionSuite2) TestPointGetStmtHints(c *C) { config.UpdateGlobal(func(conf *config.Config) { conf.OOMAction = config.OOMActionCancel }) - tk.MustExec("insert /*+ MEMORY_QUOTA(1 GB) */ into t1 values (1);") - tk.MustExec("select /*+ MEMORY_QUOTA(0 GB) */ * from t1 where a = 1;") - val := int64(0) - c.Assert(tk.Se.GetSessionVars().StmtCtx.MemTracker.CheckBytesLimit(val), IsTrue) - c.Assert(tk.Se.GetSessionVars().StmtCtx.GetWarnings()[0].Err.Error(), Equals, "Setting the MEMORY_QUOTA to 0 means no memory limit") + tk.MustExec("insert into t1 values (1);") + tk.MustExec("select * from t1 where a = 1;") tk.MustExec("set tidb_mem_quota_query=1;") err := tk.QueryToErr("select * from t1 where a = 1;") c.Assert(err, NotNil) c.Assert(err.Error(), Matches, "Out Of Memory Quota!.*") - tk.MustQuery("select /*+ MEMORY_QUOTA(1 GB) */ * from t1 where a = 1;").Check(testkit.Rows("1")) err = tk.ExecToErr("update t1 set a = a + 10 where a in (1,2,3);") + c.Assert(err, NotNil) c.Assert(err.Error(), Matches, "Out Of Memory Quota!.*") - err = tk.ExecToErr("update /*+ memory_quota(1 MB) */ t1 set a = a + 10 where a in (1,2,3);") + tk.MustExec("set tidb_mem_quota_query=1024;") + tk.MustQuery("select * from t1 where a = 1;").Check(testkit.Rows("1")) + err = tk.ExecToErr("update t1 set a = a + 10 where a in (1,2,3);") c.Check(err, IsNil) } From 93f09b19fec79e05f898b2d20fd00681ee5e6027 Mon Sep 17 00:00:00 2001 From: AI-Login-is-already-taken <956308585@qq.com> Date: Fri, 18 Dec 2020 01:09:41 +0800 Subject: [PATCH 11/18] fix --- executor/adapter.go | 6 ------ executor/batch_point_get.go | 18 +++++++++--------- executor/builder.go | 3 --- executor/point_get.go | 21 +++++++++------------ planner/core/point_get_plan.go | 2 -- session/session_test.go | 4 ++-- 6 files changed, 20 insertions(+), 34 deletions(-) diff --git a/executor/adapter.go b/executor/adapter.go index c0502333b2973..d491421d52944 100644 --- a/executor/adapter.go +++ b/executor/adapter.go @@ -748,12 +748,6 @@ func (a *ExecStmt) buildExecutor() (Executor, error) { b := newExecutorBuilder(ctx, a.InfoSchema) - if pg, ok := a.Plan.(*plannercore.PointGetPlan); ok { - pg.TrackMem = true - } - if bpg, ok := a.Plan.(*plannercore.BatchPointGetPlan); ok { - bpg.TrackMem = true - } e := b.build(a.Plan) if b.err != nil { return nil, errors.Trace(b.err) diff --git a/executor/batch_point_get.go b/executor/batch_point_get.go index 00fde2c30a240..0839651a7e4b9 100644 --- a/executor/batch_point_get.go +++ b/executor/batch_point_get.go @@ -70,8 +70,6 @@ type BatchPointGetExec struct { snapshot kv.Snapshot stats *runtimeStatsWithSnapshot - // control if track mem - trackMem bool memTracker *memory.Tracker } @@ -129,15 +127,16 @@ func (e *BatchPointGetExec) Open(context.Context) error { e.snapshot = snapshot e.batchGetter = batchGetter - if e.trackMem { - e.memTracker = memory.NewTracker(e.id, -1) - e.memTracker.AttachTo(e.ctx.GetSessionVars().StmtCtx.MemTracker) - } + e.memTracker = memory.NewTracker(e.id, -1) + e.memTracker.AttachTo(e.ctx.GetSessionVars().StmtCtx.MemTracker) return nil } // Close implements the Executor interface. func (e *BatchPointGetExec) Close() error { + //memory usage decreases + e.memTracker.Consume(-e.memTracker.BytesConsumed()) + if e.runtimeStats != nil && e.snapshot != nil { e.snapshot.DelOption(kv.CollectRuntimeStats) } @@ -163,9 +162,6 @@ func (e *BatchPointGetExec) Next(ctx context.Context, req *chunk.Chunk) error { } for !req.IsFull() && e.index < len(e.values) { handle, val := e.handles[e.index], e.values[e.index] - if e.memTracker != nil { - e.memTracker.Consume(int64(len(val))) - } err := DecodeRowValToChunk(e.base().ctx, e.schema, e.tblInfo, handle, val, req, e.rowDecoder) if err != nil { return err @@ -174,6 +170,7 @@ func (e *BatchPointGetExec) Next(ctx context.Context, req *chunk.Chunk) error { } err := FillVirtualColumnValue(e.virtualColumnRetFieldTypes, e.virtualColumnIndex, e.schema, e.columns, e.ctx, req) + e.memTracker.Consume(req.MemoryUsage()) if err != nil { return err } @@ -362,6 +359,9 @@ func (e *BatchPointGetExec) initialize(ctx context.Context) error { e.values = make([][]byte, 0, len(values)) for i, key := range keys { val := values[string(key)] + // track memory + e.memTracker.Consume(int64(len(key)) + int64(len(val))) + if len(val) == 0 { if e.idxInfo != nil && (!e.tblInfo.IsCommonHandle || !e.idxInfo.Primary) { return kv.ErrNotExist.GenWithStack("inconsistent extra index %s, handle %d not found in table", diff --git a/executor/builder.go b/executor/builder.go index 81d812a0ce8e4..d6d4506e7ad1d 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -3849,9 +3849,6 @@ func (b *executorBuilder) buildBatchPointGet(plan *plannercore.BatchPointGetPlan if e.lock { b.hasLock = true } - if plan.TrackMem { - e.trackMem = true - } var capacity int if plan.IndexInfo != nil && !isCommonHandleRead(plan.TblInfo, plan.IndexInfo) { e.idxVals = plan.IndexValues diff --git a/executor/point_get.go b/executor/point_get.go index 95fdc334ffc50..91c9cb1a1c100 100644 --- a/executor/point_get.go +++ b/executor/point_get.go @@ -50,9 +50,6 @@ func (b *executorBuilder) buildPointGet(p *plannercore.PointGetPlan) Executor { if p.Lock { b.hasLock = true } - if p.TrackMem { - e.trackMem = true - } e.Init(p, startTS) return e } @@ -86,8 +83,6 @@ type PointGetExecutor struct { stats *runtimeStatsWithSnapshot - // control if track mem - trackMem bool memTracker *memory.Tracker } @@ -149,15 +144,16 @@ func (e *PointGetExecutor) Open(context.Context) error { } e.snapshot.SetOption(kv.TaskID, e.ctx.GetSessionVars().StmtCtx.TaskID) - if e.trackMem { - e.memTracker = memory.NewTracker(e.id, -1) - e.memTracker.AttachTo(e.ctx.GetSessionVars().StmtCtx.MemTracker) - } + e.memTracker = memory.NewTracker(e.id, -1) + e.memTracker.AttachTo(e.ctx.GetSessionVars().StmtCtx.MemTracker) return nil } // Close implements the Executor interface. func (e *PointGetExecutor) Close() error { + //memory usage decreases + e.memTracker.Consume(-e.memTracker.BytesConsumed()) + if e.runtimeStats != nil && e.snapshot != nil { e.snapshot.DelOption(kv.CollectRuntimeStats) } @@ -260,9 +256,8 @@ func (e *PointGetExecutor) Next(ctx context.Context, req *chunk.Chunk) error { } return nil } - if e.memTracker != nil { - e.memTracker.Consume(int64(len(val))) - } + e.memTracker.Consume(int64(len(val))) + err = DecodeRowValToChunk(e.base().ctx, e.schema, e.tblInfo, e.handle, val, req, e.rowDecoder) if err != nil { return err @@ -270,6 +265,8 @@ func (e *PointGetExecutor) Next(ctx context.Context, req *chunk.Chunk) error { err = FillVirtualColumnValue(e.virtualColumnRetFieldTypes, e.virtualColumnIndex, e.schema, e.columns, e.ctx, req) + + e.memTracker.Consume(req.MemoryUsage()) if err != nil { return err } diff --git a/planner/core/point_get_plan.go b/planner/core/point_get_plan.go index d544c8d32e2e6..95fef18b04d59 100644 --- a/planner/core/point_get_plan.go +++ b/planner/core/point_get_plan.go @@ -67,7 +67,6 @@ type PointGetPlan struct { LockWaitTime int64 partitionColumnPos int Columns []*model.ColumnInfo - TrackMem bool } type nameValuePair struct { @@ -259,7 +258,6 @@ type BatchPointGetPlan struct { Lock bool LockWaitTime int64 Columns []*model.ColumnInfo - TrackMem bool } // Clone implements PhysicalPlan interface. diff --git a/session/session_test.go b/session/session_test.go index ab6e38d945504..8b037c3deeafc 100644 --- a/session/session_test.go +++ b/session/session_test.go @@ -3145,8 +3145,8 @@ func (s *testSessionSuite2) TestStmtHints(c *C) { c.Assert(tk.Se.GetSessionVars().GetReplicaRead(), Equals, kv.ReplicaReadFollower) } -// Test MEMORY_QUOTA hint in POINT_GET/BATCH_POINT_GET -func (s *testSessionSuite2) TestPointGetStmtHints(c *C) { +// Test memory track in POINT_GET/BATCH_POINT_GET for issue 21653 +func (s *testSessionSuite2) TestPointGetMemoryTracking(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec("drop table if exists t1;") From 5883956bad0a7810a4d1480e12ebf15ef4893977 Mon Sep 17 00:00:00 2001 From: AI-Login-is-already-taken <956308585@qq.com> Date: Sat, 26 Dec 2020 00:29:32 +0800 Subject: [PATCH 12/18] fix --- executor/adapter.go | 1 - executor/batch_point_get.go | 4 ++-- executor/point_get.go | 5 ++--- planner/optimize.go | 1 + session/session_test.go | 9 +++++---- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/executor/adapter.go b/executor/adapter.go index d491421d52944..9608609acd6c3 100644 --- a/executor/adapter.go +++ b/executor/adapter.go @@ -747,7 +747,6 @@ func (a *ExecStmt) buildExecutor() (Executor, error) { } b := newExecutorBuilder(ctx, a.InfoSchema) - e := b.build(a.Plan) if b.err != nil { return nil, errors.Trace(b.err) diff --git a/executor/batch_point_get.go b/executor/batch_point_get.go index 0839651a7e4b9..9a41a2f4bbf81 100644 --- a/executor/batch_point_get.go +++ b/executor/batch_point_get.go @@ -134,8 +134,9 @@ func (e *BatchPointGetExec) Open(context.Context) error { // Close implements the Executor interface. func (e *BatchPointGetExec) Close() error { - //memory usage decreases + e.memTracker.Consume(-e.memTracker.BytesConsumed()) + e.memTracker = nil if e.runtimeStats != nil && e.snapshot != nil { e.snapshot.DelOption(kv.CollectRuntimeStats) @@ -170,7 +171,6 @@ func (e *BatchPointGetExec) Next(ctx context.Context, req *chunk.Chunk) error { } err := FillVirtualColumnValue(e.virtualColumnRetFieldTypes, e.virtualColumnIndex, e.schema, e.columns, e.ctx, req) - e.memTracker.Consume(req.MemoryUsage()) if err != nil { return err } diff --git a/executor/point_get.go b/executor/point_get.go index 91c9cb1a1c100..2eef152f0f995 100644 --- a/executor/point_get.go +++ b/executor/point_get.go @@ -151,8 +151,9 @@ func (e *PointGetExecutor) Open(context.Context) error { // Close implements the Executor interface. func (e *PointGetExecutor) Close() error { - //memory usage decreases + e.memTracker.Consume(-e.memTracker.BytesConsumed()) + e.memTracker = nil if e.runtimeStats != nil && e.snapshot != nil { e.snapshot.DelOption(kv.CollectRuntimeStats) @@ -265,8 +266,6 @@ func (e *PointGetExecutor) Next(ctx context.Context, req *chunk.Chunk) error { err = FillVirtualColumnValue(e.virtualColumnRetFieldTypes, e.virtualColumnIndex, e.schema, e.columns, e.ctx, req) - - e.memTracker.Consume(req.MemoryUsage()) if err != nil { return err } diff --git a/planner/optimize.go b/planner/optimize.go index 2a366fc4a9298..997b8b2ca080c 100644 --- a/planner/optimize.go +++ b/planner/optimize.go @@ -86,6 +86,7 @@ func Optimize(ctx context.Context, sctx sessionctx.Context, node ast.Node, is in sessVars.IsolationReadEngines[kv.TiFlash] = struct{}{} }() } + if _, isolationReadContainTiKV := sessVars.IsolationReadEngines[kv.TiKV]; isolationReadContainTiKV { var fp plannercore.Plan if fpv, ok := sctx.Value(plannercore.PointPlanKey).(plannercore.PointPlanVal); ok { diff --git a/session/session_test.go b/session/session_test.go index 8b037c3deeafc..b5f9ea540ae7e 100644 --- a/session/session_test.go +++ b/session/session_test.go @@ -3151,22 +3151,23 @@ func (s *testSessionSuite2) TestPointGetMemoryTracking(c *C) { tk.MustExec("use test") tk.MustExec("drop table if exists t1;") tk.MustExec("create table t1(a int primary key);") - defer config.RestoreFunc()() config.UpdateGlobal(func(conf *config.Config) { conf.OOMAction = config.OOMActionCancel }) tk.MustExec("insert into t1 values (1);") - tk.MustExec("select * from t1 where a = 1;") + tk.MustQuery("select * from t1 where a = 1;").Check(testkit.Rows("1")) tk.MustExec("set tidb_mem_quota_query=1;") err := tk.QueryToErr("select * from t1 where a = 1;") c.Assert(err, NotNil) c.Assert(err.Error(), Matches, "Out Of Memory Quota!.*") - err = tk.ExecToErr("update t1 set a = a + 10 where a in (1,2,3);") + err = tk.QueryToErr("select * from t1 where a in (1,2,3);") c.Assert(err, NotNil) c.Assert(err.Error(), Matches, "Out Of Memory Quota!.*") tk.MustExec("set tidb_mem_quota_query=1024;") tk.MustQuery("select * from t1 where a = 1;").Check(testkit.Rows("1")) - err = tk.ExecToErr("update t1 set a = a + 10 where a in (1,2,3);") + err = tk.QueryToErr("select * from t1 where a = 1;") + c.Check(err, IsNil) + err = tk.QueryToErr("select * from t1 where a in (1,2,3);") c.Check(err, IsNil) } From 08d46710346c9fea67101aca900fb0cd16fd2525 Mon Sep 17 00:00:00 2001 From: AI-Login-is-already-taken <956308585@qq.com> Date: Sun, 3 Jan 2021 22:09:37 +0800 Subject: [PATCH 13/18] fix --- executor/batch_point_get.go | 5 +++-- executor/point_get.go | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/executor/batch_point_get.go b/executor/batch_point_get.go index 9a41a2f4bbf81..2864490b936a8 100644 --- a/executor/batch_point_get.go +++ b/executor/batch_point_get.go @@ -202,6 +202,7 @@ func (e *BatchPointGetExec) initialize(ctx context.Context) error { continue } + e.memTracker.Consume(types.EstimatedMemUsage(idxVals, 1)) physID := getPhysID(e.tblInfo, idxVals[e.partPos].GetInt64()) idxKey, err1 := EncodeUniqueIndexKey(e.ctx, e.tblInfo, e.idxInfo, idxVals, physID) if err1 != nil && !kv.ErrNotExist.Equal(err1) { @@ -331,6 +332,7 @@ func (e *BatchPointGetExec) initialize(ctx context.Context) error { tID = getPhysID(e.tblInfo, d.GetInt64()) } } + e.memTracker.Consume(int64(cap(handle.Encoded()))) key := tablecodec.EncodeRowKeyWithHandle(tID, handle) keys[i] = key } @@ -359,8 +361,7 @@ func (e *BatchPointGetExec) initialize(ctx context.Context) error { e.values = make([][]byte, 0, len(values)) for i, key := range keys { val := values[string(key)] - // track memory - e.memTracker.Consume(int64(len(key)) + int64(len(val))) + e.memTracker.Consume(int64(cap(val))) if len(val) == 0 { if e.idxInfo != nil && (!e.tblInfo.IsCommonHandle || !e.idxInfo.Primary) { diff --git a/executor/point_get.go b/executor/point_get.go index 2eef152f0f995..6d2503b506375 100644 --- a/executor/point_get.go +++ b/executor/point_get.go @@ -257,7 +257,7 @@ func (e *PointGetExecutor) Next(ctx context.Context, req *chunk.Chunk) error { } return nil } - e.memTracker.Consume(int64(len(val))) + e.memTracker.Consume(int64(cap(val))) err = DecodeRowValToChunk(e.base().ctx, e.schema, e.tblInfo, e.handle, val, req, e.rowDecoder) if err != nil { From 8fd842c95589db6ad0fd792bf1ffc75468c2a256 Mon Sep 17 00:00:00 2001 From: AI-Login-is-already-taken <956308585@qq.com> Date: Mon, 4 Jan 2021 08:39:43 +0800 Subject: [PATCH 14/18] fix --- executor/batch_point_get.go | 6 +++--- kv/key.go | 12 ++++++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/executor/batch_point_get.go b/executor/batch_point_get.go index 2864490b936a8..a0717ef09bf50 100644 --- a/executor/batch_point_get.go +++ b/executor/batch_point_get.go @@ -197,12 +197,12 @@ func (e *BatchPointGetExec) initialize(ctx context.Context) error { dedup := make(map[hack.MutableString]struct{}) toFetchIndexKeys := make([]kv.Key, 0, len(e.idxVals)) for _, idxVals := range e.idxVals { + e.memTracker.Consume(types.EstimatedMemUsage(idxVals, 1)) // For all x, 'x IN (null)' evaluate to null, so the query get no result. if datumsContainNull(idxVals) { continue } - e.memTracker.Consume(types.EstimatedMemUsage(idxVals, 1)) physID := getPhysID(e.tblInfo, idxVals[e.partPos].GetInt64()) idxKey, err1 := EncodeUniqueIndexKey(e.ctx, e.tblInfo, e.idxInfo, idxVals, physID) if err1 != nil && !kv.ErrNotExist.Equal(err1) { @@ -332,7 +332,6 @@ func (e *BatchPointGetExec) initialize(ctx context.Context) error { tID = getPhysID(e.tblInfo, d.GetInt64()) } } - e.memTracker.Consume(int64(cap(handle.Encoded()))) key := tablecodec.EncodeRowKeyWithHandle(tID, handle) keys[i] = key } @@ -361,7 +360,6 @@ func (e *BatchPointGetExec) initialize(ctx context.Context) error { e.values = make([][]byte, 0, len(values)) for i, key := range keys { val := values[string(key)] - e.memTracker.Consume(int64(cap(val))) if len(val) == 0 { if e.idxInfo != nil && (!e.tblInfo.IsCommonHandle || !e.idxInfo.Primary) { @@ -371,7 +369,9 @@ func (e *BatchPointGetExec) initialize(ctx context.Context) error { continue } e.values = append(e.values, val) + e.memTracker.Consume(int64(cap(val))) handles = append(handles, e.handles[i]) + e.memTracker.Consume(e.handles[i].MemoryUsage()) if e.lock && rc { existKeys = append(existKeys, key) // when e.handles is set in builder directly, index should be primary key and the plan is CommonHandleRead diff --git a/kv/key.go b/kv/key.go index 53eacf05a8a3d..0fb5509837408 100644 --- a/kv/key.go +++ b/kv/key.go @@ -151,6 +151,8 @@ type Handle interface { Data() ([]types.Datum, error) // String implements the fmt.Stringer interface. String() string + // MemoryUsage returns the total memory usage of a Handle in bytes. + MemoryUsage() int64 } // IntHandle implement the Handle interface for int64 type handle. @@ -222,6 +224,11 @@ func (ih IntHandle) String() string { return strconv.FormatInt(int64(ih), 10) } +// MemoryUsage implements the Handle interface. +func (ih *IntHandle) MemoryUsage() int64 { + return 8 +} + // CommonHandle implements the Handle interface for non-int64 type handle. type CommonHandle struct { encoded []byte @@ -341,6 +348,11 @@ func (ch *CommonHandle) String() string { return fmt.Sprintf("{%s}", strings.Join(strs, ", ")) } +// MemoryUsage implements the Handle interface. +func (ch *CommonHandle) MemoryUsage() int64 { + return int64(cap(ch.encoded)) +} + // HandleMap is the map for Handle. type HandleMap struct { ints map[int64]interface{} From 86a523a13f5fb6ce4ea0adcd08395850743b69d6 Mon Sep 17 00:00:00 2001 From: AI-Login-is-already-taken <956308585@qq.com> Date: Mon, 4 Jan 2021 08:50:59 +0800 Subject: [PATCH 15/18] fix --- kv/key.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kv/key.go b/kv/key.go index 0fb5509837408..8915f8b53e09f 100644 --- a/kv/key.go +++ b/kv/key.go @@ -225,7 +225,7 @@ func (ih IntHandle) String() string { } // MemoryUsage implements the Handle interface. -func (ih *IntHandle) MemoryUsage() int64 { +func (ih IntHandle) MemoryUsage() int64 { return 8 } From 93c21bdbded2a9aa229a4b72a52371853fa13a54 Mon Sep 17 00:00:00 2001 From: AI-Login-is-already-taken <956308585@qq.com> Date: Wed, 13 Jan 2021 22:00:30 +0800 Subject: [PATCH 16/18] fix --- executor/point_get.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/executor/point_get.go b/executor/point_get.go index 6d2503b506375..ac7d5ab59d5ed 100644 --- a/executor/point_get.go +++ b/executor/point_get.go @@ -202,6 +202,7 @@ func (e *PointGetExecutor) Next(ctx context.Context, req *chunk.Chunk) error { if err != nil && !kv.ErrNotExist.Equal(err) { return err } + e.memTracker.Consume(int64(cap(e.idxKey))) e.handleVal, err = e.get(ctx, e.idxKey) if err != nil { @@ -209,7 +210,7 @@ func (e *PointGetExecutor) Next(ctx context.Context, req *chunk.Chunk) error { return err } } - + e.memTracker.Consume(int64(cap(e.handleVal))) // try lock the index key if isolation level is not read consistency // also lock key if read consistency read a value if !e.ctx.GetSessionVars().IsPessimisticReadConsistency() || len(e.handleVal) > 0 { @@ -244,7 +245,7 @@ func (e *PointGetExecutor) Next(ctx context.Context, req *chunk.Chunk) error { }) } } - + e.memTracker.Consume(e.handle.MemoryUsage()) key := tablecodec.EncodeRowKeyWithHandle(tblID, e.handle) val, err := e.getAndLock(ctx, key) if err != nil { @@ -257,8 +258,6 @@ func (e *PointGetExecutor) Next(ctx context.Context, req *chunk.Chunk) error { } return nil } - e.memTracker.Consume(int64(cap(val))) - err = DecodeRowValToChunk(e.base().ctx, e.schema, e.tblInfo, e.handle, val, req, e.rowDecoder) if err != nil { return err From 46013d042df07eddbd145b03f8720e8226ae0d2a Mon Sep 17 00:00:00 2001 From: AI-Login-is-already-taken <956308585@qq.com> Date: Sat, 12 Feb 2022 00:35:15 +0800 Subject: [PATCH 17/18] format fix --- executor/point_get.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executor/point_get.go b/executor/point_get.go index b16dcb7ea065a..0fed01780e4dd 100644 --- a/executor/point_get.go +++ b/executor/point_get.go @@ -38,8 +38,8 @@ import ( "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/codec" "github.com/pingcap/tidb/util/execdetails" - "github.com/pingcap/tidb/util/memory" "github.com/pingcap/tidb/util/logutil/consistency" + "github.com/pingcap/tidb/util/memory" "github.com/pingcap/tidb/util/rowcodec" "github.com/tikv/client-go/v2/txnkv/txnsnapshot" ) From 47e7c0a99de42858f6e789e806e09aae7514ffd7 Mon Sep 17 00:00:00 2001 From: AI-Login-is-already-taken <956308585@qq.com> Date: Sat, 12 Feb 2022 16:01:43 +0800 Subject: [PATCH 18/18] fix bug --- executor/batch_point_get.go | 6 ++++-- executor/point_get.go | 7 ++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/executor/batch_point_get.go b/executor/batch_point_get.go index 033ab9a1bb2ef..eb413e409ae12 100644 --- a/executor/batch_point_get.go +++ b/executor/batch_point_get.go @@ -219,8 +219,10 @@ func MockNewCacheTableSnapShot(snapshot kv.Snapshot, memBuffer kv.MemBuffer) *ca // Close implements the Executor interface. func (e *BatchPointGetExec) Close() error { - e.memTracker.Consume(-e.memTracker.BytesConsumed()) - e.memTracker = nil + if e.memTracker != nil { + e.memTracker.Consume(-e.memTracker.BytesConsumed()) + e.memTracker = nil + } if e.runtimeStats != nil && e.snapshot != nil { e.snapshot.SetOption(kv.CollectRuntimeStats, nil) diff --git a/executor/point_get.go b/executor/point_get.go index 0fed01780e4dd..80b3a6f18edfc 100644 --- a/executor/point_get.go +++ b/executor/point_get.go @@ -203,9 +203,10 @@ func (e *PointGetExecutor) Open(context.Context) error { // Close implements the Executor interface. func (e *PointGetExecutor) Close() error { - - e.memTracker.Consume(-e.memTracker.BytesConsumed()) - e.memTracker = nil + if e.memTracker != nil { + e.memTracker.Consume(-e.memTracker.BytesConsumed()) + e.memTracker = nil + } if e.runtimeStats != nil && e.snapshot != nil { e.snapshot.SetOption(kv.CollectRuntimeStats, nil)