Skip to content

Commit

Permalink
Merge branch 'master' of https://github.com/pingcap/tidb into fix_pro…
Browse files Browse the repository at this point in the history 
…j_parunion
  • Loading branch information
@wjhuang2016
wjhuang2016 committed Nov 30, 2020
2 parents cdb5fab + 67f83a4 commit 0872743
Show file tree
Hide file tree
Showing 243 changed files with 9,050 additions and 2,296 deletions.
2 changes: 1 addition & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -237,7 +237,7 @@ tools/bin/failpoint-ctl: go.mod
$(GO) build -o $@ github.com/pingcap/failpoint/failpoint-ctl

tools/bin/errdoc-gen: go.mod
$(GO) build -o $@ github.com/pingcap/tiup/components/errdoc/errdoc-gen
$(GO) build -o $@ github.com/pingcap/errors/errdoc-gen

tools/bin/golangci-lint:
curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b ./tools/bin v1.29.0
Expand Down
389 changes: 387 additions & 2 deletions bindinfo/bind_test.go
View file

Large diffs are not rendered by default.

42 changes: 35 additions & 7 deletions bindinfo/handle.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -596,16 +596,19 @@ func (h *BindHandle) logicalDeleteBindInfoSQL(originalSQL, db string, updateTs t
// CaptureBaselines is used to automatically capture plan baselines.
func (h *BindHandle) CaptureBaselines() {
parser4Capture := parser.New()
schemas, sqls := stmtsummary.StmtSummaryByDigestMap.GetMoreThanOnceSelect()
schemas, sqls := stmtsummary.StmtSummaryByDigestMap.GetMoreThanOnceBindableStmt()
for i := range sqls {
stmt, err := parser4Capture.ParseOneStmt(sqls[i], "", "")
if err != nil {
logutil.BgLogger().Debug("parse SQL failed", zap.String("SQL", sqls[i]), zap.Error(err))
continue
}
normalizedSQL, digiest := parser.NormalizeDigest(sqls[i])
if insertStmt, ok := stmt.(*ast.InsertStmt); ok && insertStmt.Select == nil {
continue
}
normalizedSQL, digest := parser.NormalizeDigest(sqls[i])
dbName := utilparser.GetDefaultDB(stmt, schemas[i])
if r := h.GetBindRecord(digiest, normalizedSQL, dbName); r != nil && r.HasUsingBinding() {
if r := h.GetBindRecord(digest, normalizedSQL, dbName); r != nil && r.HasUsingBinding() {
continue
}
h.sctx.Lock()
Expand Down Expand Up @@ -682,10 +685,35 @@ func GenerateBindSQL(ctx context.Context, stmtNode ast.StmtNode, planHint string
logutil.Logger(ctx).Warn("Restore SQL failed", zap.Error(err))
}
bindSQL := sb.String()
selectIdx := strings.Index(bindSQL, "SELECT")
// Remove possible `explain` prefix.
bindSQL = bindSQL[selectIdx:]
return strings.Replace(bindSQL, "SELECT", fmt.Sprintf("SELECT /*+ %s*/", planHint), 1)
switch n := stmtNode.(type) {
case *ast.DeleteStmt:
deleteIdx := strings.Index(bindSQL, "DELETE")
// Remove possible `explain` prefix.
bindSQL = bindSQL[deleteIdx:]
return strings.Replace(bindSQL, "DELETE", fmt.Sprintf("DELETE /*+ %s*/", planHint), 1)
case *ast.UpdateStmt:
updateIdx := strings.Index(bindSQL, "UPDATE")
// Remove possible `explain` prefix.
bindSQL = bindSQL[updateIdx:]
return strings.Replace(bindSQL, "UPDATE", fmt.Sprintf("UPDATE /*+ %s*/", planHint), 1)
case *ast.SelectStmt:
selectIdx := strings.Index(bindSQL, "SELECT")
// Remove possible `explain` prefix.
bindSQL = bindSQL[selectIdx:]
return strings.Replace(bindSQL, "SELECT", fmt.Sprintf("SELECT /*+ %s*/", planHint), 1)
case *ast.InsertStmt:
insertIdx := int(0)
if n.IsReplace {
insertIdx = strings.Index(bindSQL, "REPLACE")
} else {
insertIdx = strings.Index(bindSQL, "INSERT")
}
// Remove possible `explain` prefix.
bindSQL = bindSQL[insertIdx:]
return strings.Replace(bindSQL, "SELECT", fmt.Sprintf("SELECT /*+ %s*/", planHint), 1)
}
logutil.Logger(ctx).Warn("Unexpected statement type")
return ""
}

type paramMarkerChecker struct {
Expand Down
36 changes: 18 additions & 18 deletions cmd/explaintest/r/explain_complex.result
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,28 +118,28 @@ id estRows task access object operator info
Projection_13 1.00 root test.st.id, test.dd.id, test.st.aid, test.st.cm, test.dd.dic, test.dd.ip, test.dd.t, test.st.p1, test.st.p2, test.st.p3, test.st.p4, test.st.p5, test.st.p6_md5, test.st.p7_md5, test.st.ext, test.st.t
└─Limit_16 1.00 root offset:0, count:2500
└─HashAgg_19 1.00 root group by:test.dd.dic, test.st.aid, funcs:firstrow(test.st.id)->test.st.id, funcs:firstrow(test.st.aid)->test.st.aid, funcs:firstrow(test.st.cm)->test.st.cm, funcs:firstrow(test.st.p1)->test.st.p1, funcs:firstrow(test.st.p2)->test.st.p2, funcs:firstrow(test.st.p3)->test.st.p3, funcs:firstrow(test.st.p4)->test.st.p4, funcs:firstrow(test.st.p5)->test.st.p5, funcs:firstrow(test.st.p6_md5)->test.st.p6_md5, funcs:firstrow(test.st.p7_md5)->test.st.p7_md5, funcs:firstrow(test.st.ext)->test.st.ext, funcs:firstrow(test.st.t)->test.st.t, funcs:firstrow(test.dd.id)->test.dd.id, funcs:firstrow(test.dd.dic)->test.dd.dic, funcs:firstrow(test.dd.ip)->test.dd.ip, funcs:firstrow(test.dd.t)->test.dd.t
└─HashJoin_34 0.00 root inner join, equal:[eq(test.dd.aid, test.st.aid) eq(test.dd.ip, test.st.ip)], other cond:gt(test.dd.t, test.st.t)
├─IndexLookUp_52(Build) 0.00 root
│ ├─IndexRangeScan_49(Build) 3333.33 cop[tikv] table:dd, index:t(t) range:(1478143908,+inf], keep order:false, stats:pseudo
│ └─Selection_51(Probe) 0.00 cop[tikv] eq(test.dd.bm, 0), eq(test.dd.pt, "android"), not(isnull(test.dd.ip))
│ └─TableRowIDScan_50 3333.33 cop[tikv] table:dd keep order:false, stats:pseudo
└─IndexLookUp_41(Probe) 3.33 root
├─IndexRangeScan_38(Build) 3333.33 cop[tikv] table:gad, index:t(t) range:(1478143908,+inf], keep order:false, stats:pseudo
└─Selection_40(Probe) 3.33 cop[tikv] eq(test.st.pt, "android"), not(isnull(test.st.ip))
└─TableRowIDScan_39 3333.33 cop[tikv] table:gad keep order:false, stats:pseudo
└─HashJoin_33 0.00 root inner join, equal:[eq(test.dd.aid, test.st.aid) eq(test.dd.ip, test.st.ip)], other cond:gt(test.dd.t, test.st.t)
├─IndexLookUp_51(Build) 0.00 root
│ ├─IndexRangeScan_48(Build) 3333.33 cop[tikv] table:dd, index:t(t) range:(1478143908,+inf], keep order:false, stats:pseudo
│ └─Selection_50(Probe) 0.00 cop[tikv] eq(test.dd.bm, 0), eq(test.dd.pt, "android"), not(isnull(test.dd.ip))
│ └─TableRowIDScan_49 3333.33 cop[tikv] table:dd keep order:false, stats:pseudo
└─IndexLookUp_40(Probe) 3.33 root
├─IndexRangeScan_37(Build) 3333.33 cop[tikv] table:gad, index:t(t) range:(1478143908,+inf], keep order:false, stats:pseudo
└─Selection_39(Probe) 3.33 cop[tikv] eq(test.st.pt, "android"), not(isnull(test.st.ip))
└─TableRowIDScan_38 3333.33 cop[tikv] table:gad keep order:false, stats:pseudo
explain select gad.id as gid,sdk.id as sid,gad.aid as aid,gad.cm as cm,sdk.dic as dic,sdk.ip as ip, sdk.t as t, gad.p1 as p1, gad.p2 as p2, gad.p3 as p3, gad.p4 as p4, gad.p5 as p5, gad.p6_md5 as p6, gad.p7_md5 as p7, gad.ext as ext from st gad join dd sdk on gad.aid = sdk.aid and gad.dic = sdk.mac and gad.t < sdk.t where gad.t > 1477971479 and gad.bm = 0 and gad.pt = 'ios' and gad.dit = 'mac' and sdk.t > 1477971479 and sdk.bm = 0 and sdk.pt = 'ios' limit 3000;
id estRows task access object operator info
Projection_10 0.00 root test.st.id, test.dd.id, test.st.aid, test.st.cm, test.dd.dic, test.dd.ip, test.dd.t, test.st.p1, test.st.p2, test.st.p3, test.st.p4, test.st.p5, test.st.p6_md5, test.st.p7_md5, test.st.ext
└─Limit_13 0.00 root offset:0, count:3000
└─IndexMergeJoin_26 0.00 root inner join, inner:IndexLookUp_24, outer key:test.st.aid, inner key:test.dd.aid, other cond:eq(test.st.dic, test.dd.mac), lt(test.st.t, test.dd.t)
├─IndexLookUp_35(Build) 0.00 root
│ ├─IndexRangeScan_32(Build) 3333.33 cop[tikv] table:gad, index:t(t) range:(1477971479,+inf], keep order:false, stats:pseudo
│ └─Selection_34(Probe) 0.00 cop[tikv] eq(test.st.bm, 0), eq(test.st.dit, "mac"), eq(test.st.pt, "ios"), not(isnull(test.st.dic))
│ └─TableRowIDScan_33 3333.33 cop[tikv] table:gad keep order:false, stats:pseudo
└─IndexLookUp_24(Probe) 0.00 root
├─IndexRangeScan_21(Build) 10000.00 cop[tikv] table:sdk, index:aid(aid, dic) range: decided by [eq(test.dd.aid, test.st.aid)], keep order:true, stats:pseudo
└─Selection_23(Probe) 0.00 cop[tikv] eq(test.dd.bm, 0), eq(test.dd.pt, "ios"), gt(test.dd.t, 1477971479), not(isnull(test.dd.mac)), not(isnull(test.dd.t))
└─TableRowIDScan_22 10000.00 cop[tikv] table:sdk keep order:false, stats:pseudo
└─IndexJoin_18 0.00 root inner join, inner:IndexLookUp_17, outer key:test.st.aid, inner key:test.dd.aid, equal cond:eq(test.st.aid, test.dd.aid), eq(test.st.dic, test.dd.mac), other cond:lt(test.st.t, test.dd.t)
├─IndexLookUp_34(Build) 0.00 root
│ ├─IndexRangeScan_31(Build) 3333.33 cop[tikv] table:gad, index:t(t) range:(1477971479,+inf], keep order:false, stats:pseudo
│ └─Selection_33(Probe) 0.00 cop[tikv] eq(test.st.bm, 0), eq(test.st.dit, "mac"), eq(test.st.pt, "ios"), not(isnull(test.st.dic))
│ └─TableRowIDScan_32 3333.33 cop[tikv] table:gad keep order:false, stats:pseudo
└─IndexLookUp_17(Probe) 0.00 root
├─IndexRangeScan_14(Build) 10000.00 cop[tikv] table:sdk, index:aid(aid, dic) range: decided by [eq(test.dd.aid, test.st.aid)], keep order:false, stats:pseudo
└─Selection_16(Probe) 0.00 cop[tikv] eq(test.dd.bm, 0), eq(test.dd.pt, "ios"), gt(test.dd.t, 1477971479), not(isnull(test.dd.mac)), not(isnull(test.dd.t))
└─TableRowIDScan_15 10000.00 cop[tikv] table:sdk keep order:false, stats:pseudo
explain SELECT cm, p1, p2, p3, p4, p5, p6_md5, p7_md5, count(1) as click_pv, count(DISTINCT ip) as click_ip FROM st WHERE (t between 1478188800 and 1478275200) and aid='cn.sbkcq' and pt='android' GROUP BY cm, p1, p2, p3, p4, p5, p6_md5, p7_md5;
id estRows task access object operator info
Projection_5 1.00 root test.st.cm, test.st.p1, test.st.p2, test.st.p3, test.st.p4, test.st.p5, test.st.p6_md5, test.st.p7_md5, Column#20, Column#21
Expand Down
30 changes: 15 additions & 15 deletions cmd/explaintest/r/explain_complex_stats.result
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,25 +128,25 @@ id estRows task access object operator info
Projection_13 424.00 root test.st.id, test.dd.id, test.st.aid, test.st.cm, test.dd.dic, test.dd.ip, test.dd.t, test.st.p1, test.st.p2, test.st.p3, test.st.p4, test.st.p5, test.st.p6_md5, test.st.p7_md5, test.st.ext, test.st.t
└─Limit_16 424.00 root offset:0, count:2500
└─HashAgg_19 424.00 root group by:test.dd.dic, test.st.aid, funcs:firstrow(test.st.id)->test.st.id, funcs:firstrow(test.st.aid)->test.st.aid, funcs:firstrow(test.st.cm)->test.st.cm, funcs:firstrow(test.st.p1)->test.st.p1, funcs:firstrow(test.st.p2)->test.st.p2, funcs:firstrow(test.st.p3)->test.st.p3, funcs:firstrow(test.st.p4)->test.st.p4, funcs:firstrow(test.st.p5)->test.st.p5, funcs:firstrow(test.st.p6_md5)->test.st.p6_md5, funcs:firstrow(test.st.p7_md5)->test.st.p7_md5, funcs:firstrow(test.st.ext)->test.st.ext, funcs:firstrow(test.st.t)->test.st.t, funcs:firstrow(test.dd.id)->test.dd.id, funcs:firstrow(test.dd.dic)->test.dd.dic, funcs:firstrow(test.dd.ip)->test.dd.ip, funcs:firstrow(test.dd.t)->test.dd.t
└─HashJoin_34 424.00 root inner join, equal:[eq(test.st.aid, test.dd.aid) eq(test.st.ip, test.dd.ip)], other cond:gt(test.dd.t, test.st.t)
├─TableReader_37(Build) 424.00 root data:Selection_36
│ └─Selection_36 424.00 cop[tikv] eq(test.st.bm, 0), eq(test.st.pt, "android"), gt(test.st.t, 1478143908), not(isnull(test.st.ip))
│ └─TableRangeScan_35 1999.00 cop[tikv] table:gad range:[0,+inf], keep order:false
└─TableReader_44(Probe) 455.80 root data:Selection_43
└─Selection_43 455.80 cop[tikv] eq(test.dd.bm, 0), eq(test.dd.pt, "android"), gt(test.dd.t, 1478143908), not(isnull(test.dd.ip)), not(isnull(test.dd.t))
└─TableRangeScan_42 2000.00 cop[tikv] table:dd range:[0,+inf], keep order:false
└─HashJoin_33 424.00 root inner join, equal:[eq(test.st.aid, test.dd.aid) eq(test.st.ip, test.dd.ip)], other cond:gt(test.dd.t, test.st.t)
├─TableReader_36(Build) 424.00 root data:Selection_35
│ └─Selection_35 424.00 cop[tikv] eq(test.st.bm, 0), eq(test.st.pt, "android"), gt(test.st.t, 1478143908), not(isnull(test.st.ip))
│ └─TableRangeScan_34 1999.00 cop[tikv] table:gad range:[0,+inf], keep order:false
└─TableReader_43(Probe) 455.80 root data:Selection_42
└─Selection_42 455.80 cop[tikv] eq(test.dd.bm, 0), eq(test.dd.pt, "android"), gt(test.dd.t, 1478143908), not(isnull(test.dd.ip)), not(isnull(test.dd.t))
└─TableRangeScan_41 2000.00 cop[tikv] table:dd range:[0,+inf], keep order:false
explain select gad.id as gid,sdk.id as sid,gad.aid as aid,gad.cm as cm,sdk.dic as dic,sdk.ip as ip, sdk.t as t, gad.p1 as p1, gad.p2 as p2, gad.p3 as p3, gad.p4 as p4, gad.p5 as p5, gad.p6_md5 as p6, gad.p7_md5 as p7, gad.ext as ext from st gad join dd sdk on gad.aid = sdk.aid and gad.dic = sdk.mac and gad.t < sdk.t where gad.t > 1477971479 and gad.bm = 0 and gad.pt = 'ios' and gad.dit = 'mac' and sdk.t > 1477971479 and sdk.bm = 0 and sdk.pt = 'ios' limit 3000;
id estRows task access object operator info
Projection_10 170.34 root test.st.id, test.dd.id, test.st.aid, test.st.cm, test.dd.dic, test.dd.ip, test.dd.t, test.st.p1, test.st.p2, test.st.p3, test.st.p4, test.st.p5, test.st.p6_md5, test.st.p7_md5, test.st.ext
└─Limit_13 170.34 root offset:0, count:3000
└─IndexMergeJoin_26 170.34 root inner join, inner:IndexLookUp_24, outer key:test.st.aid, inner key:test.dd.aid, other cond:eq(test.st.dic, test.dd.mac), lt(test.st.t, test.dd.t)
├─TableReader_31(Build) 170.34 root data:Selection_30
│ └─Selection_30 170.34 cop[tikv] eq(test.st.bm, 0), eq(test.st.dit, "mac"), eq(test.st.pt, "ios"), gt(test.st.t, 1477971479), not(isnull(test.st.dic))
│ └─TableRangeScan_29 1999.00 cop[tikv] table:gad range:[0,+inf], keep order:false
└─IndexLookUp_24(Probe) 1.00 root
├─IndexRangeScan_21(Build) 3.93 cop[tikv] table:sdk, index:aid(aid, dic) range: decided by [eq(test.dd.aid, test.st.aid)], keep order:true
└─Selection_23(Probe) 1.00 cop[tikv] eq(test.dd.bm, 0), eq(test.dd.pt, "ios"), gt(test.dd.t, 1477971479), not(isnull(test.dd.mac)), not(isnull(test.dd.t))
└─TableRowIDScan_22 3.93 cop[tikv] table:sdk keep order:false
└─IndexJoin_18 170.34 root inner join, inner:IndexLookUp_17, outer key:test.st.aid, inner key:test.dd.aid, equal cond:eq(test.st.aid, test.dd.aid), eq(test.st.dic, test.dd.mac), other cond:lt(test.st.t, test.dd.t)
├─TableReader_30(Build) 170.34 root data:Selection_29
│ └─Selection_29 170.34 cop[tikv] eq(test.st.bm, 0), eq(test.st.dit, "mac"), eq(test.st.pt, "ios"), gt(test.st.t, 1477971479), not(isnull(test.st.dic))
│ └─TableRangeScan_28 1999.00 cop[tikv] table:gad range:[0,+inf], keep order:false
└─IndexLookUp_17(Probe) 1.00 root
├─IndexRangeScan_14(Build) 3.93 cop[tikv] table:sdk, index:aid(aid, dic) range: decided by [eq(test.dd.aid, test.st.aid)], keep order:false
└─Selection_16(Probe) 1.00 cop[tikv] eq(test.dd.bm, 0), eq(test.dd.pt, "ios"), gt(test.dd.t, 1477971479), not(isnull(test.dd.mac)), not(isnull(test.dd.t))
└─TableRowIDScan_15 3.93 cop[tikv] table:sdk keep order:false
explain SELECT cm, p1, p2, p3, p4, p5, p6_md5, p7_md5, count(1) as click_pv, count(DISTINCT ip) as click_ip FROM st WHERE (t between 1478188800 and 1478275200) and aid='cn.sbkcq' and pt='android' GROUP BY cm, p1, p2, p3, p4, p5, p6_md5, p7_md5;
id estRows task access object operator info
Projection_5 39.28 root test.st.cm, test.st.p1, test.st.p2, test.st.p3, test.st.p4, test.st.p5, test.st.p6_md5, test.st.p7_md5, Column#20, Column#21
Expand Down
Loading

0 comments on commit 0872743

Please sign in to comment.