Skip to content

Commit

Permalink
planner: support trace candidates for Datasource (#31810)
Browse files Browse the repository at this point in the history 
ref #29661
  • Loading branch information
@Yisaer
Yisaer authored Jan 21, 2022
1 parent ad1c5b5 commit 2d62d23
Show file tree
Hide file tree
Showing 8 changed files with 131 additions and 122 deletions.
2 changes: 1 addition & 1 deletion executor/executor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1705,7 +1705,7 @@ func ResetContextOfStmt(ctx sessionctx.Context, s ast.StmtNode) (err error) {
sc.IsStaleness = false
sc.LockTableIDs = make(map[int64]struct{})
sc.EnableOptimizeTrace = false
sc.LogicalOptimizeTrace = nil
sc.OptimizeTracer = nil
sc.OptimizerCETrace = nil

sc.InitMemTracker(memory.LabelForSQLText, vars.MemQuotaQuery)
Expand Down
5 changes: 1 addition & 4 deletions executor/trace.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@ import (
"github.com/pingcap/tidb/util/chunk"
"github.com/pingcap/tidb/util/logutil"
"github.com/pingcap/tidb/util/sqlexec"
"github.com/pingcap/tidb/util/tracing"
"go.uber.org/zap"
"sourcegraph.com/sourcegraph/appdash"
traceImpl "sourcegraph.com/sourcegraph/appdash/opentracing"
Expand Down Expand Up @@ -162,9 +161,7 @@ func (e *TraceExec) nextOptimizerPlanTrace(ctx context.Context, se sessionctx.Co
jsonEncoder := json.NewEncoder(&writer)
// If we do not set this to false, ">", "<", "&"... will be escaped to "\u003c","\u003e", "\u0026"...
jsonEncoder.SetEscapeHTML(false)
logical := se.GetSessionVars().StmtCtx.LogicalOptimizeTrace
physical := se.GetSessionVars().StmtCtx.PhysicalOptimizeTrace
err = jsonEncoder.Encode(&tracing.OptimizeTracer{Logical: logical, Physical: physical})
err = jsonEncoder.Encode(se.GetSessionVars().StmtCtx.OptimizeTracer)
if err != nil {
return errors.AddStack(err)
}
Expand Down
96 changes: 44 additions & 52 deletions planner/core/find_best_task.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,6 @@ func (p *baseLogicalPlan) rebuildChildTasks(childTasks *[]task, pp PhysicalPlan,
func (p *baseLogicalPlan) enumeratePhysicalPlans4Task(physicalPlans []PhysicalPlan, prop *property.PhysicalProperty, addEnforcer bool, planCounter *PlanCounterTp, opt *physicalOptimizeOp) (task, int64, error) {
var bestTask task = invalidTask
var curCntPlan, cntPlan int64
childProps := make(map[task]*property.PhysicalProperty)
childTasks := make([]task, 0, len(p.children))
childCnts := make([]int64, len(p.children))
cntPlan = 0
Expand All @@ -231,7 +230,6 @@ func (p *baseLogicalPlan) enumeratePhysicalPlans4Task(physicalPlans []PhysicalPl
break
}
childTasks = append(childTasks, childTask)
childProps[childTask] = childProp
}

// This check makes sure that there is no invalid child task.
Expand Down Expand Up @@ -279,15 +277,7 @@ func (p *baseLogicalPlan) enumeratePhysicalPlans4Task(physicalPlans []PhysicalPl
bestTask = curTask
break
}
c := opt.appendCandidate(p, curTask.plan(), prop)
if c != nil {
for _, childTask := range childTasks {
if childTask == nil {
continue
}
opt.appendChildToCandidate(c, childTask.plan(), childProps[childTask])
}
}
opt.appendCandidate(p, curTask.plan(), prop)
// Get the most efficient one.
if curTask.cost() < bestTask.cost() || (bestTask.invalid() && !curTask.invalid()) {
bestTask = curTask
Expand All @@ -310,44 +300,30 @@ func (op *physicalOptimizeOp) withEnableOptimizeTracer(tracer *tracing.PhysicalO
return op
}

func (op *physicalOptimizeOp) buildPhysicalOptimizeTraceInfo(p LogicalPlan, prop string) *tracing.PhysicalOptimizeTraceInfo {
func (op *physicalOptimizeOp) buildPhysicalOptimizeTraceInfo(p LogicalPlan) {
if op == nil || op.tracer == nil {
return nil
return
}
name := tracing.CodecPlanName(p.TP(), p.ID())
if _, ok := op.tracer.State[name]; !ok {
op.tracer.State[name] = make(map[string]*tracing.PhysicalOptimizeTraceInfo)
}
if info, ok := op.tracer.State[name][prop]; ok {
return info
op.tracer.State[name] = make(map[string]*tracing.PlanTrace)
}
traceInfo := &tracing.PhysicalOptimizeTraceInfo{Property: prop}
op.tracer.State[name][prop] = traceInfo
return traceInfo
}

func (op *physicalOptimizeOp) appendChildToCandidate(candidateInfo *tracing.PlanTrace, plan PhysicalPlan, prop *property.PhysicalProperty) {
if op == nil || op.tracer == nil || candidateInfo == nil {
func (op *physicalOptimizeOp) appendCandidate(lp LogicalPlan, pp PhysicalPlan, prop *property.PhysicalProperty) {
if op == nil || op.tracer == nil || pp == nil {
return
}
childPhysicalPlanTrace := &tracing.PlanTrace{TP: plan.TP(), ID: plan.ID(), ExplainInfo: plan.ExplainInfo(), Cost: plan.Cost(), ProperType: prop.String()}
candidateInfo.Children = append(candidateInfo.Children, childPhysicalPlanTrace)
}

func (op *physicalOptimizeOp) appendCandidate(logicalPlan LogicalPlan, physicalPlan PhysicalPlan, prop *property.PhysicalProperty) *tracing.PlanTrace {
if op == nil || op.tracer == nil {
return nil
}
key := string(prop.HashCode())
PhysicalPlanTrace := &tracing.PlanTrace{TP: physicalPlan.TP(), ID: physicalPlan.ID(),
ExplainInfo: physicalPlan.ExplainInfo(), Cost: physicalPlan.Cost(), ProperType: prop.String()}
name := tracing.CodecPlanName(logicalPlan.TP(), logicalPlan.ID())
traceInfo := op.tracer.State[name][key]
if traceInfo == nil {
traceInfo = op.buildPhysicalOptimizeTraceInfo(logicalPlan, key)
PhysicalPlanTrace := &tracing.PlanTrace{TP: pp.TP(), ID: pp.ID(),
ExplainInfo: pp.ExplainInfo(), Cost: pp.Cost(), ProperType: prop.String()}
name := tracing.CodecPlanName(lp.TP(), lp.ID())
key := tracing.CodecPlanName(pp.TP(), pp.ID())
pps := op.tracer.State[name]
if pps == nil {
op.buildPhysicalOptimizeTraceInfo(lp)
}
traceInfo.Candidates = append(traceInfo.Candidates, PhysicalPlanTrace)
return PhysicalPlanTrace
pps[key] = PhysicalPlanTrace
op.tracer.State[name] = pps
}

// findBestTask implements LogicalPlan interface.
Expand Down Expand Up @@ -421,7 +397,7 @@ func (p *baseLogicalPlan) findBestTask(prop *property.PhysicalProperty, planCoun

var cnt int64
var curTask task
opt.buildPhysicalOptimizeTraceInfo(p, newProp.String())
opt.buildPhysicalOptimizeTraceInfo(p)
if bestTask, cnt, err = p.enumeratePhysicalPlans4Task(plansFitsProp, newProp, false, planCounter, opt); err != nil {
return nil, 0, err
}
Expand All @@ -439,6 +415,7 @@ func (p *baseLogicalPlan) findBestTask(prop *property.PhysicalProperty, planCoun
bestTask = curTask
goto END
}
opt.appendCandidate(p, curTask.plan(), prop)
if curTask.cost() < bestTask.cost() || (bestTask.invalid() && !curTask.invalid()) {
bestTask = curTask
}
Expand Down Expand Up @@ -831,18 +808,20 @@ func (ds *DataSource) findBestTask(prop *property.PhysicalProperty, planCounter
}
}()

opt.buildPhysicalOptimizeTraceInfo(ds)
cntPlan = 0
for _, candidate := range candidates {
path := candidate.path
if path.PartialIndexPaths != nil {
idxMergeTask, err := ds.convertToIndexMergeScan(prop, candidate)
idxMergeTask, err := ds.convertToIndexMergeScan(prop, candidate, opt)
if err != nil {
return nil, 0, err
}
if !idxMergeTask.invalid() {
cntPlan += 1
planCounter.Dec(1)
}
appendCandidate(ds, idxMergeTask, prop, opt)
if idxMergeTask.cost() < t.cost() || planCounter.Empty() {
t = idxMergeTask
}
Expand Down Expand Up @@ -922,10 +901,11 @@ func (ds *DataSource) findBestTask(prop *property.PhysicalProperty, planCounter
if allRangeIsPoint {
var pointGetTask task
if len(path.Ranges) == 1 {
pointGetTask = ds.convertToPointGet(prop, candidate)
pointGetTask = ds.convertToPointGet(prop, candidate, opt)
} else {
pointGetTask = ds.convertToBatchPointGet(prop, candidate, hashPartColName)
pointGetTask = ds.convertToBatchPointGet(prop, candidate, hashPartColName, opt)
}
appendCandidate(ds, pointGetTask, prop, opt)
if !pointGetTask.invalid() {
cntPlan += 1
planCounter.Dec(1)
Expand All @@ -948,9 +928,9 @@ func (ds *DataSource) findBestTask(prop *property.PhysicalProperty, planCounter
}
var tblTask task
if ds.SampleInfo != nil {
tblTask, err = ds.convertToSampleTable(prop, candidate)
tblTask, err = ds.convertToSampleTable(prop, candidate, opt)
} else {
tblTask, err = ds.convertToTableScan(prop, candidate)
tblTask, err = ds.convertToTableScan(prop, candidate, opt)
}
if err != nil {
return nil, 0, err
Expand All @@ -959,6 +939,7 @@ func (ds *DataSource) findBestTask(prop *property.PhysicalProperty, planCounter
cntPlan += 1
planCounter.Dec(1)
}
appendCandidate(ds, tblTask, prop, opt)
if tblTask.cost() < t.cost() || planCounter.Empty() {
t = tblTask
}
Expand All @@ -971,14 +952,15 @@ func (ds *DataSource) findBestTask(prop *property.PhysicalProperty, planCounter
if ds.preferStoreType&preferTiFlash != 0 {
continue
}
idxTask, err := ds.convertToIndexScan(prop, candidate)
idxTask, err := ds.convertToIndexScan(prop, candidate, opt)
if err != nil {
return nil, 0, err
}
if !idxTask.invalid() {
cntPlan += 1
planCounter.Dec(1)
}
appendCandidate(ds, idxTask, prop, opt)
if idxTask.cost() < t.cost() || planCounter.Empty() {
t = idxTask
}
Expand Down Expand Up @@ -1011,7 +993,7 @@ func (ds *DataSource) canConvertToPointGetForPlanCache(path *util.AccessPath) bo
return false
}

func (ds *DataSource) convertToIndexMergeScan(prop *property.PhysicalProperty, candidate *candidatePath) (task task, err error) {
func (ds *DataSource) convertToIndexMergeScan(prop *property.PhysicalProperty, candidate *candidatePath, opt *physicalOptimizeOp) (task task, err error) {
if prop.TaskTp != property.RootTaskType || !prop.IsEmpty() {
return invalidTask, nil
}
Expand Down Expand Up @@ -1299,7 +1281,8 @@ func (ds *DataSource) addSelection4PlanCache(task *rootTask, stats *property.Sta
}

// convertToIndexScan converts the DataSource to index scan with idx.
func (ds *DataSource) convertToIndexScan(prop *property.PhysicalProperty, candidate *candidatePath) (task task, err error) {
func (ds *DataSource) convertToIndexScan(prop *property.PhysicalProperty,
candidate *candidatePath, opt *physicalOptimizeOp) (task task, err error) {
if !candidate.path.IsSingleScan {
// If it's parent requires single read task, return max cost.
if prop.TaskTp == property.CopSingleReadTaskType {
Expand Down Expand Up @@ -1805,7 +1788,7 @@ func (s *LogicalIndexScan) GetPhysicalIndexScan(schema *expression.Schema, stats
}

// convertToTableScan converts the DataSource to table scan.
func (ds *DataSource) convertToTableScan(prop *property.PhysicalProperty, candidate *candidatePath) (task task, err error) {
func (ds *DataSource) convertToTableScan(prop *property.PhysicalProperty, candidate *candidatePath, opt *physicalOptimizeOp) (task task, err error) {
// It will be handled in convertToIndexScan.
if prop.TaskTp == property.CopDoubleReadTaskType {
return invalidTask, nil
Expand Down Expand Up @@ -1882,7 +1865,8 @@ func (ds *DataSource) convertToTableScan(prop *property.PhysicalProperty, candid
return task, nil
}

func (ds *DataSource) convertToSampleTable(prop *property.PhysicalProperty, candidate *candidatePath) (task task, err error) {
func (ds *DataSource) convertToSampleTable(prop *property.PhysicalProperty,
candidate *candidatePath, opt *physicalOptimizeOp) (task task, err error) {
if prop.TaskTp == property.CopDoubleReadTaskType {
return invalidTask, nil
}
Expand All @@ -1906,7 +1890,7 @@ func (ds *DataSource) convertToSampleTable(prop *property.PhysicalProperty, cand
}, nil
}

func (ds *DataSource) convertToPointGet(prop *property.PhysicalProperty, candidate *candidatePath) task {
func (ds *DataSource) convertToPointGet(prop *property.PhysicalProperty, candidate *candidatePath, opt *physicalOptimizeOp) (task task) {
if !prop.IsEmpty() && !candidate.isMatchProp {
return invalidTask
}
Expand Down Expand Up @@ -1990,7 +1974,8 @@ func (ds *DataSource) convertToPointGet(prop *property.PhysicalProperty, candida
return rTsk
}

func (ds *DataSource) convertToBatchPointGet(prop *property.PhysicalProperty, candidate *candidatePath, hashPartColName *ast.ColumnName) task {
func (ds *DataSource) convertToBatchPointGet(prop *property.PhysicalProperty,
candidate *candidatePath, hashPartColName *ast.ColumnName, opt *physicalOptimizeOp) (task task) {
if !prop.IsEmpty() && !candidate.isMatchProp {
return invalidTask
}
Expand Down Expand Up @@ -2266,3 +2251,10 @@ func (p *LogicalCTETable) findBestTask(prop *property.PhysicalProperty, planCoun
t = &rootTask{p: pcteTable}
return t, 1, nil
}

func appendCandidate(lp LogicalPlan, task task, prop *property.PhysicalProperty, opt *physicalOptimizeOp) {
if task == nil || task.invalid() {
return
}
opt.appendCandidate(lp, task.plan(), prop)
}
2 changes: 1 addition & 1 deletion planner/core/logical_plan_trace_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -414,7 +414,7 @@ func (s *testPlanSuite) TestSingleRuleTraceStep(c *C) {
}
_, err = logicalOptimize(ctx, flag, p.(LogicalPlan))
c.Assert(err, IsNil)
otrace := sctx.GetSessionVars().StmtCtx.LogicalOptimizeTrace
otrace := sctx.GetSessionVars().StmtCtx.OptimizeTracer.Logical
c.Assert(otrace, NotNil)
assert := false
for _, step := range otrace.Steps {
Expand Down
11 changes: 7 additions & 4 deletions planner/core/optimizer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -290,7 +290,9 @@ func DoOptimize(ctx context.Context, sctx sessionctx.Context, flag uint64, logic
if sctx.GetSessionVars().StmtCtx.EnableOptimizerCETrace {
refineCETrace(sctx)
}

if sctx.GetSessionVars().StmtCtx.EnableOptimizeTrace {
sctx.GetSessionVars().StmtCtx.OptimizeTracer.RecordFinalPlan(finalPlan.buildPlanTrace())
}
return finalPlan, cost, nil
}

Expand Down Expand Up @@ -409,12 +411,13 @@ func logicalOptimize(ctx context.Context, flag uint64, logic LogicalPlan) (Logic
opt := defaultLogicalOptimizeOption()
vars := logic.SCtx().GetSessionVars()
if vars.StmtCtx.EnableOptimizeTrace {
vars.StmtCtx.OptimizeTracer = &tracing.OptimizeTracer{}
tracer := &tracing.LogicalOptimizeTracer{
Steps: make([]*tracing.LogicalRuleOptimizeTracer, 0),
}
opt = opt.withEnableOptimizeTracer(tracer)
defer func() {
vars.StmtCtx.LogicalOptimizeTrace = tracer
vars.StmtCtx.OptimizeTracer.Logical = tracer
}()
}
var err error
Expand Down Expand Up @@ -455,12 +458,12 @@ func physicalOptimize(logic LogicalPlan, planCounter *PlanCounterTp) (plan Physi
opt := defaultPhysicalOptimizeOption()
stmtCtx := logic.SCtx().GetSessionVars().StmtCtx
if stmtCtx.EnableOptimizeTrace {
tracer := &tracing.PhysicalOptimizeTracer{State: make(map[string]map[string]*tracing.PhysicalOptimizeTraceInfo)}
tracer := &tracing.PhysicalOptimizeTracer{State: make(map[string]map[string]*tracing.PlanTrace)}
opt = opt.withEnableOptimizeTracer(tracer)
defer func() {
if err == nil {
tracer.RecordFinalPlanTrace(plan.buildPlanTrace())
stmtCtx.PhysicalOptimizeTrace = tracer
stmtCtx.OptimizeTracer.Physical = tracer
}
}()
}
Expand Down
Loading

0 comments on commit 2d62d23

Please sign in to comment.