From 25c69ac6f060930da6f4cb513e671efaeebf5195 Mon Sep 17 00:00:00 2001 From: yisaer Date: Tue, 23 Nov 2021 17:13:01 +0800 Subject: [PATCH 1/7] support dump trace plan Signed-off-by: yisaer --- executor/builder.go | 11 +++-- executor/executor.go | 1 - executor/trace.go | 79 +++++++++++++++++++++++++++++++ planner/core/logical_plan_test.go | 2 +- planner/core/optimizer.go | 6 +-- planner/core/planbuilder.go | 9 +++- planner/core/trace.go | 3 ++ sessionctx/stmtctx/stmtctx.go | 2 - sessionctx/variable/session.go | 3 ++ util/tracing/opt_trace.go | 26 +++++----- 10 files changed, 116 insertions(+), 26 deletions(-) diff --git a/executor/builder.go b/executor/builder.go index c277eb8e91b79..e92894786005b 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -973,12 +973,13 @@ func (b *executorBuilder) buildDDL(v *plannercore.DDL) Executor { // at build(). func (b *executorBuilder) buildTrace(v *plannercore.Trace) Executor { t := &TraceExec{ - baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ID()), - stmtNode: v.StmtNode, - builder: b, - format: v.Format, + baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ID()), + stmtNode: v.StmtNode, + builder: b, + format: v.Format, + optimizerTrace: v.OptimizerTrace, } - if t.format == plannercore.TraceFormatLog { + if t.format == plannercore.TraceFormatLog && !t.optimizerTrace { return &SortExec{ baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ID(), t), ByItems: []*plannerutil.ByItems{ diff --git a/executor/executor.go b/executor/executor.go index 7383941acd046..7daacd2e04c24 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -1682,7 +1682,6 @@ func ResetContextOfStmt(ctx sessionctx.Context, s ast.StmtNode) (err error) { sc.CTEStorageMap = map[int]*CTEStorages{} sc.IsStaleness = false sc.LockTableIDs = make(map[int64]struct{}) - sc.EnableOptimizeTrace = false sc.LogicalOptimizeTrace = nil sc.InitMemTracker(memory.LabelForSQLText, vars.MemQuotaQuery) diff --git a/executor/trace.go b/executor/trace.go index e9a1e6bc91a81..57feabbca78e9 100644 --- a/executor/trace.go +++ b/executor/trace.go @@ -15,10 +15,16 @@ package executor import ( + "archive/zip" "context" + "encoding/base64" "encoding/json" "fmt" + "math/rand" + "os" + "path/filepath" "sort" + "strconv" "time" "github.com/opentracing/basictracer-go" @@ -51,6 +57,8 @@ type TraceExec struct { builder *executorBuilder format string + // optimizerTrace indicates 'trace plan statement' + optimizerTrace bool } // Next executes real query and collects span later. @@ -71,6 +79,10 @@ func (e *TraceExec) Next(ctx context.Context, req *chunk.Chunk) error { e.ctx.GetSessionVars().StmtCtx = stmtCtx }() + if e.optimizerTrace { + return e.nextOptimizerPlanTrace(ctx, e.ctx, req) + } + switch e.format { case core.TraceFormatLog: return e.nextTraceLog(ctx, se, req) @@ -79,6 +91,40 @@ func (e *TraceExec) Next(ctx context.Context, req *chunk.Chunk) error { } } +func (e *TraceExec) nextOptimizerPlanTrace(ctx context.Context, se sessionctx.Context, req *chunk.Chunk) error { + zf, fileName, err := generateOptimizerTraceFile() + if err != nil { + return err + } + zw := zip.NewWriter(zf) + defer func() { + err := zw.Close() + if err != nil { + logutil.BgLogger().Warn("Closing zip writer failed", zap.Error(err)) + } + err = zf.Close() + if err != nil { + logutil.BgLogger().Warn("Closing zip file failed", zap.Error(err)) + } + }() + traceZW, err := zw.Create("trace.json") + if err != nil { + return errors.AddStack(err) + } + e.executeChild(ctx, se.(sqlexec.SQLExecutor)) + res, err := json.Marshal(se.GetSessionVars().StmtCtx.LogicalOptimizeTrace) + if err != nil { + return errors.AddStack(err) + } + _, err = traceZW.Write(res) + if err != nil { + return errors.AddStack(err) + } + req.AppendString(0, fileName) + e.exhausted = true + return nil +} + func (e *TraceExec) nextTraceLog(ctx context.Context, se sqlexec.SQLExecutor, req *chunk.Chunk) error { recorder := basictracer.NewInMemoryRecorder() tracer := basictracer.New(recorder) @@ -142,8 +188,11 @@ func (e *TraceExec) executeChild(ctx context.Context, se sqlexec.SQLExecutor) { vars := e.ctx.GetSessionVars() origin := vars.InRestrictedSQL vars.InRestrictedSQL = true + originOptimizeTrace := vars.EnableStmtOptimizeTrace + vars.EnableStmtOptimizeTrace = e.optimizerTrace defer func() { vars.InRestrictedSQL = origin + vars.EnableStmtOptimizeTrace = originOptimizeTrace }() rs, err := se.ExecuteStmt(ctx, e.stmtNode) if err != nil { @@ -252,3 +301,33 @@ func generateLogResult(allSpans []basictracer.RawSpan, chk *chunk.Chunk) { } } } + +func generateOptimizerTraceFile() (*os.File, string, error) { + dirPath := getOptimizerTraceDirName() + // Create path + err := os.MkdirAll(dirPath, os.ModePerm) + if err != nil { + return nil, "", errors.AddStack(err) + } + // Generate key and create zip file + time := time.Now().UnixNano() + b := make([]byte, 16) + _, err = rand.Read(b) + if err != nil { + return nil, "", errors.AddStack(err) + } + key := base64.URLEncoding.EncodeToString(b) + fileName := fmt.Sprintf("optimizer_trace_%v_%v.zip", key, time) + zf, err := os.Create(filepath.Join(dirPath, fileName)) + fmt.Println(filepath.Join(dirPath, fileName)) + if err != nil { + return nil, "", errors.AddStack(err) + } + return zf, fileName, nil +} + +// getOptimizerTraceDirName returns optimizer trace directory path. +// The path is related to the process id. +func getOptimizerTraceDirName() string { + return filepath.Join(os.TempDir(), "optimizer_trace", strconv.Itoa(os.Getpid())) +} diff --git a/planner/core/logical_plan_test.go b/planner/core/logical_plan_test.go index c800f5c577a4f..b662db288d3b1 100644 --- a/planner/core/logical_plan_test.go +++ b/planner/core/logical_plan_test.go @@ -2107,7 +2107,7 @@ func (s *testPlanSuite) TestLogicalOptimizeWithTraceEnabled(c *C) { err = Preprocess(s.ctx, stmt, WithPreprocessorReturn(&PreprocessorReturn{InfoSchema: s.is})) c.Assert(err, IsNil, comment) sctx := MockContext() - sctx.GetSessionVars().StmtCtx.EnableOptimizeTrace = true + sctx.GetSessionVars().EnableStmtOptimizeTrace = true builder, _ := NewPlanBuilder().Init(sctx, s.is, &hint.BlockHintProcessor{}) domain.GetDomain(sctx).MockInfoCacheAndLoadInfoSchema(s.is) ctx := context.TODO() diff --git a/planner/core/optimizer.go b/planner/core/optimizer.go index b964818245966..1b712fd090017 100644 --- a/planner/core/optimizer.go +++ b/planner/core/optimizer.go @@ -377,12 +377,12 @@ func enableParallelApply(sctx sessionctx.Context, plan PhysicalPlan) PhysicalPla func logicalOptimize(ctx context.Context, flag uint64, logic LogicalPlan) (LogicalPlan, error) { opt := defaultLogicalOptimizeOption() - stmtCtx := logic.SCtx().GetSessionVars().StmtCtx - if stmtCtx.EnableOptimizeTrace { + vars := logic.SCtx().GetSessionVars() + if vars.EnableStmtOptimizeTrace { tracer := &tracing.LogicalOptimizeTracer{Steps: make([]*tracing.LogicalRuleOptimizeTracer, 0)} opt = opt.withEnableOptimizeTracer(tracer) defer func() { - stmtCtx.LogicalOptimizeTrace = tracer + vars.StmtCtx.LogicalOptimizeTrace = tracer }() } var err error diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index d1fc3ba01cdc3..85ca461accca1 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -4048,7 +4048,14 @@ const ( // underlying query and then constructs a schema, which will be used to constructs // rows result. func (b *PlanBuilder) buildTrace(trace *ast.TraceStmt) (Plan, error) { - p := &Trace{StmtNode: trace.Stmt, Format: trace.Format} + p := &Trace{StmtNode: trace.Stmt, Format: trace.Format, OptimizerTrace: trace.TracePlan} + if trace.TracePlan { + schema := newColumnsWithNames(1) + schema.Append(buildColumnWithName("", "Dump_link", mysql.TypeVarchar, 128)) + p.SetSchema(schema.col2Schema()) + p.names = schema.names + return p, nil + } switch trace.Format { case TraceFormatRow: schema := newColumnsWithNames(3) diff --git a/planner/core/trace.go b/planner/core/trace.go index 29ed57539d479..29d756cf5d4ca 100644 --- a/planner/core/trace.go +++ b/planner/core/trace.go @@ -24,4 +24,7 @@ type Trace struct { StmtNode ast.StmtNode Format string + + // OptimizerTrace indicates `trace plan ` case + OptimizerTrace bool } diff --git a/sessionctx/stmtctx/stmtctx.go b/sessionctx/stmtctx/stmtctx.go index 03c488c883a0c..d7a543f08213b 100644 --- a/sessionctx/stmtctx/stmtctx.go +++ b/sessionctx/stmtctx/stmtctx.go @@ -193,8 +193,6 @@ type StatementContext struct { // InVerboseExplain indicates the statement is "explain format='verbose' ...". InVerboseExplain bool - // EnableOptimizeTrace indicates whether the statement is enable optimize trace - EnableOptimizeTrace bool // LogicalOptimizeTrace indicates the trace for optimize LogicalOptimizeTrace *tracing.LogicalOptimizeTracer } diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index c395d98c85356..5e0b1af1e6906 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -955,6 +955,9 @@ type SessionVars struct { curr int8 data [2]stmtctx.StatementContext } + + // EnableStmtOptimizeTrace indicates whether enable optimizer trace by 'trace plan statement' + EnableStmtOptimizeTrace bool } // InitStatementContext initializes a StatementContext, the object is reused to reduce allocation. diff --git a/util/tracing/opt_trace.go b/util/tracing/opt_trace.go index 6c0e7243bcdf2..80e6dcad4ee25 100644 --- a/util/tracing/opt_trace.go +++ b/util/tracing/opt_trace.go @@ -16,17 +16,17 @@ package tracing // LogicalPlanTrace indicates for the LogicalPlan trace information type LogicalPlanTrace struct { - ID int - TP string - Children []*LogicalPlanTrace + ID int `json:"id"` + TP string `json:"type"` + Children []*LogicalPlanTrace `json:"children"` // ExplainInfo should be implemented by each implemented LogicalPlan - ExplainInfo string + ExplainInfo string `json:"info"` } // LogicalOptimizeTracer indicates the trace for the whole logicalOptimize processing type LogicalOptimizeTracer struct { - Steps []*LogicalRuleOptimizeTracer + Steps []*LogicalRuleOptimizeTracer `json:"steps"` // curRuleTracer indicates the current rule Tracer during optimize by rule curRuleTracer *LogicalRuleOptimizeTracer } @@ -56,10 +56,10 @@ func (tracer *LogicalOptimizeTracer) TrackLogicalPlanAfterRuleOptimize(after *Lo // LogicalRuleOptimizeTracer indicates the trace for the LogicalPlan tree before and after // logical rule optimize type LogicalRuleOptimizeTracer struct { - Before *LogicalPlanTrace - After *LogicalPlanTrace - RuleName string - Steps []LogicalRuleOptimizeTraceStep + Before *LogicalPlanTrace `json:"before"` + After *LogicalPlanTrace `json:"after"` + RuleName string `json:"name"` + Steps []LogicalRuleOptimizeTraceStep `json:"steps"` } // buildLogicalRuleOptimizeTracerBeforeOptimize build rule tracer before rule optimize @@ -74,8 +74,8 @@ func buildLogicalRuleOptimizeTracerBeforeOptimize(name string, before *LogicalPl // LogicalRuleOptimizeTraceStep indicates the trace for the detailed optimize changing in // logical rule optimize type LogicalRuleOptimizeTraceStep struct { - Action string - Reason string - ID int - TP string + Action string `json:"action"` + Reason string `json:"reason"` + ID int `json:"id"` + TP string `json:"type"` } From d107bb895234132e11f19bb1c63345d9143af6fc Mon Sep 17 00:00:00 2001 From: yisaer Date: Wed, 24 Nov 2021 11:44:39 +0800 Subject: [PATCH 2/7] fix Signed-off-by: yisaer --- executor/trace.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executor/trace.go b/executor/trace.go index 57feabbca78e9..eb2bfb9a74f56 100644 --- a/executor/trace.go +++ b/executor/trace.go @@ -17,10 +17,10 @@ package executor import ( "archive/zip" "context" + "crypto/rand" "encoding/base64" "encoding/json" "fmt" - "math/rand" "os" "path/filepath" "sort" From 504d46cd81dd7ef35209045d78e0a7aee3be64bf Mon Sep 17 00:00:00 2001 From: yisaer Date: Wed, 24 Nov 2021 12:51:21 +0800 Subject: [PATCH 3/7] add test Signed-off-by: yisaer --- executor/trace.go | 1 - executor/trace_test.go | 10 ++++++++++ planner/core/optimizer.go | 6 +++--- planner/core/planbuilder.go | 1 + util/tracing/opt_trace.go | 8 +++++--- 5 files changed, 19 insertions(+), 7 deletions(-) diff --git a/executor/trace.go b/executor/trace.go index eb2bfb9a74f56..98d66ca697065 100644 --- a/executor/trace.go +++ b/executor/trace.go @@ -319,7 +319,6 @@ func generateOptimizerTraceFile() (*os.File, string, error) { key := base64.URLEncoding.EncodeToString(b) fileName := fmt.Sprintf("optimizer_trace_%v_%v.zip", key, time) zf, err := os.Create(filepath.Join(dirPath, fileName)) - fmt.Println(filepath.Join(dirPath, fileName)) if err != nil { return nil, "", errors.AddStack(err) } diff --git a/executor/trace_test.go b/executor/trace_test.go index c360ee82f0856..0771151c63e2a 100644 --- a/executor/trace_test.go +++ b/executor/trace_test.go @@ -66,3 +66,13 @@ func rowsOrdered(rows [][]interface{}) bool { } return true } + +func (s *testSuite1) TestTracePlanStmt(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("create table tp123(id int);") + rows := tk.MustQuery("trace plan select * from tp123").Rows() + c.Assert(rows, HasLen, 1) + c.Assert(rows[0], HasLen, 1) + c.Assert(rows[0][0].(string), Matches, ".*zip") +} diff --git a/planner/core/optimizer.go b/planner/core/optimizer.go index 1b712fd090017..8677f16e47871 100644 --- a/planner/core/optimizer.go +++ b/planner/core/optimizer.go @@ -98,11 +98,11 @@ func (op *logicalOptimizeOp) withEnableOptimizeTracer(tracer *tracing.LogicalOpt return op } -func (op *logicalOptimizeOp) appendBeforeRuleOptimize(name string, before LogicalPlan) { +func (op *logicalOptimizeOp) appendBeforeRuleOptimize(index int, name string, before LogicalPlan) { if op.tracer == nil { return } - op.tracer.AppendRuleTracerBeforeRuleOptimize(name, before.buildLogicalPlanTrace()) + op.tracer.AppendRuleTracerBeforeRuleOptimize(index, name, before.buildLogicalPlanTrace()) } func (op *logicalOptimizeOp) appendStepToCurrent(id int, tp, reason, action string) { @@ -393,7 +393,7 @@ func logicalOptimize(ctx context.Context, flag uint64, logic LogicalPlan) (Logic if flag&(1< Date: Thu, 25 Nov 2021 15:47:35 +0800 Subject: [PATCH 4/7] support download file Signed-off-by: yisaer --- domain/optimize_trace.go | 27 +++++++++++ executor/trace.go | 10 +--- server/http_status.go | 2 + server/optimize_trace.go | 61 +++++++++++++++++++++++ server/optimize_trace_test.go | 91 +++++++++++++++++++++++++++++++++++ 5 files changed, 183 insertions(+), 8 deletions(-) create mode 100644 domain/optimize_trace.go create mode 100644 server/optimize_trace.go create mode 100644 server/optimize_trace_test.go diff --git a/domain/optimize_trace.go b/domain/optimize_trace.go new file mode 100644 index 0000000000000..f00e2416aba6d --- /dev/null +++ b/domain/optimize_trace.go @@ -0,0 +1,27 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package domain + +import ( + "os" + "path/filepath" + "strconv" +) + +// GetOptimizerTraceDirName returns optimizer trace directory path. +// The path is related to the process id. +func GetOptimizerTraceDirName() string { + return filepath.Join(os.TempDir(), "optimizer_trace", strconv.Itoa(os.Getpid())) +} diff --git a/executor/trace.go b/executor/trace.go index 98d66ca697065..b0303a643819b 100644 --- a/executor/trace.go +++ b/executor/trace.go @@ -24,12 +24,12 @@ import ( "os" "path/filepath" "sort" - "strconv" "time" "github.com/opentracing/basictracer-go" "github.com/opentracing/opentracing-go" "github.com/pingcap/errors" + "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/parser/ast" "github.com/pingcap/tidb/parser/mysql" "github.com/pingcap/tidb/parser/terror" @@ -303,7 +303,7 @@ func generateLogResult(allSpans []basictracer.RawSpan, chk *chunk.Chunk) { } func generateOptimizerTraceFile() (*os.File, string, error) { - dirPath := getOptimizerTraceDirName() + dirPath := domain.GetOptimizerTraceDirName() // Create path err := os.MkdirAll(dirPath, os.ModePerm) if err != nil { @@ -324,9 +324,3 @@ func generateOptimizerTraceFile() (*os.File, string, error) { } return zf, fileName, nil } - -// getOptimizerTraceDirName returns optimizer trace directory path. -// The path is related to the process id. -func getOptimizerTraceDirName() string { - return filepath.Join(os.TempDir(), "optimizer_trace", strconv.Itoa(os.Getpid())) -} diff --git a/server/http_status.go b/server/http_status.go index 6b29d9b92d423..8248e7c54e3de 100644 --- a/server/http_status.go +++ b/server/http_status.go @@ -203,6 +203,8 @@ func (s *Server) startHTTPServer() { router.Handle("/plan_replayer/dump/{filename}", s.newPlanReplayerHandler()).Name("PlanReplayerDump") + router.Handle("/optimize_trace/dump/{filename}", s.newOptimizeTraceHandler()).Name("OptimizeTraceDump") + tikvHandlerTool := s.newTikvHandlerTool() router.Handle("/settings", settingsHandler{tikvHandlerTool}).Name("Settings") router.Handle("/binlog/recover", binlogRecover{}).Name("BinlogRecover") diff --git a/server/optimize_trace.go b/server/optimize_trace.go new file mode 100644 index 0000000000000..8849c9eae4268 --- /dev/null +++ b/server/optimize_trace.go @@ -0,0 +1,61 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "io" + "net/http" + "os" + "path/filepath" + + "github.com/gorilla/mux" + "github.com/pingcap/errors" + "github.com/pingcap/tidb/domain" + "github.com/pingcap/tidb/parser/terror" +) + +type OptimizeTraceHandler struct { +} + +func (s *Server) newOptimizeTraceHandler() *OptimizeTraceHandler { + return &OptimizeTraceHandler{} +} + +func (prh OptimizeTraceHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { + w.Header().Set("Content-Type", "application/zip") + w.Header().Set("Content-Disposition", "attachment; filename=\"optimize_trace.zip\"") + + params := mux.Vars(req) + + name := params[pFileName] + path := filepath.Join(domain.GetOptimizerTraceDirName(), name) + file, err := os.Open(path) + if err != nil { + writeError(w, err) + } else { + _, err := io.Copy(w, file) + if err != nil { + terror.Log(errors.Trace(err)) + } + } + err = file.Close() + if err != nil { + terror.Log(errors.Trace(err)) + } + err = os.Remove(path) + if err != nil { + terror.Log(errors.Trace(err)) + } +} diff --git a/server/optimize_trace_test.go b/server/optimize_trace_test.go new file mode 100644 index 0000000000000..4ed0a2a0e0826 --- /dev/null +++ b/server/optimize_trace_test.go @@ -0,0 +1,91 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "database/sql" + "net/http" + "path/filepath" + "testing" + + "github.com/gorilla/mux" + "github.com/pingcap/tidb/session" + "github.com/pingcap/tidb/testkit" + "github.com/stretchr/testify/require" +) + +func TestDumpOptimizeTraceAPI(t *testing.T) { + store, clean := testkit.CreateMockStore(t) + defer clean() + + driver := NewTiDBDriver(store) + client := newTestServerClient() + cfg := newTestConfig() + cfg.Port = client.port + cfg.Status.StatusPort = client.statusPort + cfg.Status.ReportStatus = true + + server, err := NewServer(cfg, driver) + require.NoError(t, err) + defer server.Close() + + client.port = getPortFromTCPAddr(server.listener.Addr()) + client.statusPort = getPortFromTCPAddr(server.statusListener.Addr()) + go func() { + err := server.Run() + require.NoError(t, err) + }() + client.waitUntilServerOnline() + + dom, err := session.GetDomain(store) + require.NoError(t, err) + statsHandler := &StatsHandler{dom} + + otHandler := &OptimizeTraceHandler{} + filename := prepareData4OptimizeTrace(t, client, statsHandler) + + router := mux.NewRouter() + router.Handle("/optimize_trace/dump/{filename}", otHandler) + + resp0, err := client.fetchStatus(filepath.Join("/optimize_trace/dump/", filename)) + require.NoError(t, err) + defer func() { + require.NoError(t, resp0.Body.Close()) + }() + require.Equal(t, http.StatusOK, resp0.StatusCode) +} + +func prepareData4OptimizeTrace(t *testing.T, client *testServerClient, statHandle *StatsHandler) string { + db, err := sql.Open("mysql", client.getDSN()) + require.NoError(t, err, "Error connecting") + defer func() { + err := db.Close() + require.NoError(t, err) + }() + tk := testkit.NewDBTestKit(t, db) + + h := statHandle.do.StatsHandle() + tk.MustExec("create database optimizeTrace") + tk.MustExec("use optimizeTrace") + tk.MustExec("create table t(a int)") + err = h.HandleDDLEvent(<-h.DDLEventCh()) + require.NoError(t, err) + rows := tk.MustQuery("trace plan select * from t") + require.True(t, rows.Next(), "unexpected data") + var filename string + err = rows.Scan(&filename) + require.NoError(t, err) + return filename +} From 5078872613eeee9b070c979ec48dd7df2852e1d6 Mon Sep 17 00:00:00 2001 From: yisaer Date: Thu, 25 Nov 2021 17:01:15 +0800 Subject: [PATCH 5/7] support dump Signed-off-by: yisaer support dump Signed-off-by: yisaer --- server/optimize_trace.go | 52 +++++++++++++++++++--------------------- server/plan_replayer.go | 40 +++++++++++++++++++++++++------ 2 files changed, 58 insertions(+), 34 deletions(-) diff --git a/server/optimize_trace.go b/server/optimize_trace.go index 8849c9eae4268..351a9f6787cbe 100644 --- a/server/optimize_trace.go +++ b/server/optimize_trace.go @@ -15,47 +15,45 @@ package server import ( - "io" + "fmt" "net/http" - "os" "path/filepath" "github.com/gorilla/mux" - "github.com/pingcap/errors" + "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/domain" - "github.com/pingcap/tidb/parser/terror" + "github.com/pingcap/tidb/domain/infosync" ) type OptimizeTraceHandler struct { + infoGetter *infosync.InfoSyncer + address string + statusPort uint } func (s *Server) newOptimizeTraceHandler() *OptimizeTraceHandler { - return &OptimizeTraceHandler{} + cfg := config.GetGlobalConfig() + oth := &OptimizeTraceHandler{ + address: cfg.AdvertiseAddress, + statusPort: cfg.Status.StatusPort, + } + if s.dom != nil && s.dom.InfoSyncer() != nil { + oth.infoGetter = s.dom.InfoSyncer() + } + return oth } -func (prh OptimizeTraceHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { - w.Header().Set("Content-Type", "application/zip") - w.Header().Set("Content-Disposition", "attachment; filename=\"optimize_trace.zip\"") - +func (oth OptimizeTraceHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { params := mux.Vars(req) - name := params[pFileName] - path := filepath.Join(domain.GetOptimizerTraceDirName(), name) - file, err := os.Open(path) - if err != nil { - writeError(w, err) - } else { - _, err := io.Copy(w, file) - if err != nil { - terror.Log(errors.Trace(err)) - } - } - err = file.Close() - if err != nil { - terror.Log(errors.Trace(err)) - } - err = os.Remove(path) - if err != nil { - terror.Log(errors.Trace(err)) + handler := downloadFileHandler{ + filePath: filepath.Join(domain.GetOptimizerTraceDirName(), name), + fileName: name, + infoGetter: oth.infoGetter, + address: oth.address, + statusPort: oth.statusPort, + urlPath: fmt.Sprintf("optimize_trace/dump/%s", name), + downloadedFilename: "optimize_trace", } + handleDownloadFile(handler, w, req) } diff --git a/server/plan_replayer.go b/server/plan_replayer.go index 78f0129c084f6..8740c890b619b 100644 --- a/server/plan_replayer.go +++ b/server/plan_replayer.go @@ -53,10 +53,26 @@ func (s *Server) newPlanReplayerHandler() *PlanReplayerHandler { func (prh PlanReplayerHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { params := mux.Vars(req) name := params[pFileName] - path := filepath.Join(domain.GetPlanReplayerDirName(), name) + handler := downloadFileHandler{ + filePath: filepath.Join(domain.GetPlanReplayerDirName(), name), + fileName: name, + infoGetter: prh.infoGetter, + address: prh.address, + statusPort: prh.statusPort, + urlPath: fmt.Sprintf("plan_replyaer/dump/%s", name), + downloadedFilename: "plan_replayer", + } + handleDownloadFile(handler, w, req) +} + +func handleDownloadFile(handler downloadFileHandler, w http.ResponseWriter, req *http.Request) { + params := mux.Vars(req) + name := params[pFileName] + path := handler.filePath if isExists(path) { + logutil.BgLogger().Info("path", zap.String("path", path)) w.Header().Set("Content-Type", "application/zip") - w.Header().Set("Content-Disposition", "attachment; filename=\"plan_replayer.zip\"") + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s.zip\"", handler.downloadedFilename)) file, err := os.Open(path) if err != nil { writeError(w, err) @@ -80,7 +96,7 @@ func (prh PlanReplayerHandler) ServeHTTP(w http.ResponseWriter, req *http.Reques w.WriteHeader(http.StatusOK) return } - if prh.infoGetter == nil { + if handler.infoGetter == nil { w.WriteHeader(http.StatusNotFound) return } @@ -91,17 +107,17 @@ func (prh PlanReplayerHandler) ServeHTTP(w http.ResponseWriter, req *http.Reques return } // If we didn't find file in origin request, try to broadcast the request to all remote tidb-servers - topos, err := prh.infoGetter.GetAllTiDBTopology(req.Context()) + topos, err := handler.infoGetter.GetAllTiDBTopology(req.Context()) if err != nil { writeError(w, err) return } // transfer each remote tidb-server and try to find dump file for _, topo := range topos { - if topo.IP == prh.address && topo.StatusPort == prh.statusPort { + if topo.IP == handler.address && topo.StatusPort == handler.statusPort { continue } - url := fmt.Sprintf("http://%s:%v/plan_replayer/dump/%s?forward=true", topo.IP, topo.StatusPort, name) + url := fmt.Sprintf("http://%s:%v/%s?forward=true", topo.IP, topo.StatusPort, handler.urlPath) resp, err := http.Get(url) // #nosec G107 if err != nil { terror.Log(errors.Trace(err)) @@ -113,7 +129,7 @@ func (prh PlanReplayerHandler) ServeHTTP(w http.ResponseWriter, req *http.Reques } // find dump file in one remote tidb-server, return file directly w.Header().Set("Content-Type", "application/zip") - w.Header().Set("Content-Disposition", "attachment; filename=\"plan_replayer.zip\"") + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s.zip\"", handler.downloadedFilename)) _, err = io.Copy(w, resp.Body) if err != nil { writeError(w, err) @@ -132,6 +148,16 @@ func (prh PlanReplayerHandler) ServeHTTP(w http.ResponseWriter, req *http.Reques w.WriteHeader(http.StatusNotFound) } +type downloadFileHandler struct { + filePath string + fileName string + infoGetter *infosync.InfoSyncer + address string + statusPort uint + urlPath string + downloadedFilename string +} + func isExists(path string) bool { _, err := os.Stat(path) if err != nil && !os.IsExist(err) { From 632c3a0d25cc53a55d74eebabeebc9430fba05dc Mon Sep 17 00:00:00 2001 From: yisaer Date: Thu, 25 Nov 2021 17:03:38 +0800 Subject: [PATCH 6/7] support dump Signed-off-by: yisaer --- server/plan_replayer.go | 1 - 1 file changed, 1 deletion(-) diff --git a/server/plan_replayer.go b/server/plan_replayer.go index 8740c890b619b..a363781f0b8c9 100644 --- a/server/plan_replayer.go +++ b/server/plan_replayer.go @@ -70,7 +70,6 @@ func handleDownloadFile(handler downloadFileHandler, w http.ResponseWriter, req name := params[pFileName] path := handler.filePath if isExists(path) { - logutil.BgLogger().Info("path", zap.String("path", path)) w.Header().Set("Content-Type", "application/zip") w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s.zip\"", handler.downloadedFilename)) file, err := os.Open(path) From e362dae74c41027a9bbdfe2dbe2402fbed406989 Mon Sep 17 00:00:00 2001 From: yisaer Date: Fri, 26 Nov 2021 11:41:16 +0800 Subject: [PATCH 7/7] fix Signed-off-by: yisaer --- server/optimize_trace.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/optimize_trace.go b/server/optimize_trace.go index 351a9f6787cbe..dcd0184fd87e9 100644 --- a/server/optimize_trace.go +++ b/server/optimize_trace.go @@ -25,6 +25,7 @@ import ( "github.com/pingcap/tidb/domain/infosync" ) +// OptimizeTraceHandler serve http type OptimizeTraceHandler struct { infoGetter *infosync.InfoSyncer address string