disttask/ddl: fail task for panic (pingcap#48135)

close pingcap#48136
ti-chi-bot · Nov 1, 2023 · 5c2c01a · 5c2c01a
1 parent 8524952
commit 5c2c01a
Show file tree

Hide file tree

Showing 2 changed files with 26 additions and 0 deletions.
diff --git a/pkg/ddl/backfilling_operators.go b/pkg/ddl/backfilling_operators.go
@@ -36,6 +36,7 @@ import (
 	util2 "github.com/pingcap/tidb/pkg/ddl/util"
 	"github.com/pingcap/tidb/pkg/disttask/operator"
 	"github.com/pingcap/tidb/pkg/kv"
+	"github.com/pingcap/tidb/pkg/metrics"
 	"github.com/pingcap/tidb/pkg/parser/model"
 	"github.com/pingcap/tidb/pkg/parser/terror"
 	"github.com/pingcap/tidb/pkg/resourcemanager/pool/workerpool"
@@ -45,6 +46,7 @@ import (
 	"github.com/pingcap/tidb/pkg/table"
 	"github.com/pingcap/tidb/pkg/table/tables"
 	"github.com/pingcap/tidb/pkg/tablecodec"
+	tidbutil "github.com/pingcap/tidb/pkg/util"
 	"github.com/pingcap/tidb/pkg/util/chunk"
 	"github.com/pingcap/tidb/pkg/util/logutil"
 	"github.com/pingcap/tidb/pkg/util/memory"
@@ -402,6 +404,13 @@ type tableScanWorker struct {
 }
 
 func (w *tableScanWorker) HandleTask(task TableScanTask, sender func(IndexRecordChunk)) {
+	defer tidbutil.Recover(metrics.LblAddIndex, "handleTableScanTaskWithRecover", func() {
+		w.ctx.onError(errors.New("met panic in tableScanWorker"))
+	}, false)
+
+	failpoint.Inject("injectPanicForTableScan", func() {
+		panic("mock panic")
+	})
 	if w.se == nil {
 		sessCtx, err := w.sessPool.Get()
 		if err != nil {
@@ -604,6 +613,14 @@ func (w *indexIngestWorker) HandleTask(rs IndexRecordChunk, send func(IndexWrite
 			w.srcChunkPool <- rs.Chunk
 		}
 	}()
+	defer tidbutil.Recover(metrics.LblAddIndex, "handleIndexIngtestTaskWithRecover", func() {
+		w.ctx.onError(errors.New("met panic in indexIngestWorker"))
+	}, false)
+
+	failpoint.Inject("injectPanicForIndexIngest", func() {
+		panic("mock panic")
+	})
+
 	result := IndexWriteResult{
 		ID: rs.ID,
 	}

diff --git a/tests/realtikvtest/addindextest1/disttask_test.go b/tests/realtikvtest/addindextest1/disttask_test.go
@@ -66,6 +66,15 @@ func TestAddIndexDistBasic(t *testing.T) {
 	tk.MustExec("alter table t1 add index idx1(a);")
 	tk.MustExec("admin check index t1 idx1;")
 	require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/disttask/framework/scheduler/MockRunSubtaskContextCanceled"))
+
+	require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/injectPanicForTableScan", "return()"))
+	tk.MustExecToErr("alter table t1 add index idx2(a);")
+	require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/injectPanicForTableScan"))
+
+	require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/injectPanicForIndexIngest", "return()"))
+	tk.MustExecToErr("alter table t1 add index idx2(a);")
+	require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/injectPanicForIndexIngest"))
+
 	tk.MustExec(`set global tidb_enable_dist_task=0;`)
 }