disttask/ddl: fail task for panic (#48135)

close #48136
pingcap · Nov 1, 2023 · e30753e · e30753e
1 parent 4667ed9
commit e30753e
Show file tree

Hide file tree

Showing 2 changed files with 26 additions and 0 deletions.
diff --git a/pkg/ddl/backfilling_operators.go b/pkg/ddl/backfilling_operators.go
@@ -36,6 +36,7 @@ import (
  util2 "github.com/pingcap/tidb/pkg/ddl/util"
  "github.com/pingcap/tidb/pkg/disttask/operator"
  "github.com/pingcap/tidb/pkg/kv"
+ "github.com/pingcap/tidb/pkg/metrics"
  "github.com/pingcap/tidb/pkg/parser/model"
  "github.com/pingcap/tidb/pkg/parser/terror"
  "github.com/pingcap/tidb/pkg/resourcemanager/pool/workerpool"
@@ -45,6 +46,7 @@ import (
  "github.com/pingcap/tidb/pkg/table"
  "github.com/pingcap/tidb/pkg/table/tables"
  "github.com/pingcap/tidb/pkg/tablecodec"
+ tidbutil "github.com/pingcap/tidb/pkg/util"
  "github.com/pingcap/tidb/pkg/util/chunk"
  "github.com/pingcap/tidb/pkg/util/logutil"
  "github.com/pingcap/tidb/pkg/util/memory"
@@ -402,6 +404,13 @@ type tableScanWorker struct {
 }
 
 func (w *tableScanWorker) HandleTask(task TableScanTask, sender func(IndexRecordChunk)) {
+ defer tidbutil.Recover(metrics.LblAddIndex, "handleTableScanTaskWithRecover", func() {
+ w.ctx.onError(errors.New("met panic in tableScanWorker"))
+ }, false)
+
+ failpoint.Inject("injectPanicForTableScan", func() {
+ panic("mock panic")
+ })
  if w.se == nil {
  sessCtx, err := w.sessPool.Get()
  if err != nil {
@@ -604,6 +613,14 @@ func (w *indexIngestWorker) HandleTask(rs IndexRecordChunk, send func(IndexWrite
  w.srcChunkPool <- rs.Chunk
  }
  }()
+ defer tidbutil.Recover(metrics.LblAddIndex, "handleIndexIngtestTaskWithRecover", func() {
+ w.ctx.onError(errors.New("met panic in indexIngestWorker"))
+ }, false)
+
+ failpoint.Inject("injectPanicForIndexIngest", func() {
+ panic("mock panic")
+ })
+
  result := IndexWriteResult{
  ID: rs.ID,
  }

diff --git a/tests/realtikvtest/addindextest1/disttask_test.go b/tests/realtikvtest/addindextest1/disttask_test.go
@@ -66,6 +66,15 @@ func TestAddIndexDistBasic(t *testing.T) {
  tk.MustExec("alter table t1 add index idx1(a);")
  tk.MustExec("admin check index t1 idx1;")
  require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/disttask/framework/scheduler/MockRunSubtaskContextCanceled"))
+
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/injectPanicForTableScan", "return()"))
+ tk.MustExecToErr("alter table t1 add index idx2(a);")
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/injectPanicForTableScan"))
+
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/injectPanicForIndexIngest", "return()"))
+ tk.MustExecToErr("alter table t1 add index idx2(a);")
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/injectPanicForIndexIngest"))
+
  tk.MustExec(`set global tidb_enable_dist_task=0;`)
 }