tidb panic when execute delete statement

[yibin87]

commit hash: c131521
Log info:
[2023/11/09 05:26:43.253 +08:00] [ERROR] [conn.go:965] ["connection running loop panic"] [conn=3330277890] [session_alias=] [lastSQL="DELETE FROM new_order WHERE (no_w_id, no_d_id, no_o_id) IN ( (?,?,?),(?,?,?),(?,?,?),(?,?,?),(?,?,?),(?,?,?),(?,?,?),(?,?,?),(?,?,?),(?,?,?) ) [arguments: (74, 1, 2401, 74, 2, 2401, 74, 3, 2401, 74, 4, 2401, 74, 5, 2401, 74, 6, 2401, 74, 7, 2401, 74, 8, 2401, 74, 9, 2401, 74, 10, 2401)]"] [err="runtime error: index out of range [3] with length 3"] [stack="github.com/pingcap/tidb/pkg/server.(*clientConn).Run.func1\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/server/conn.go:968\nruntime.gopanic\n\t/usr/local/go/src/runtime/panic.go:914\ngithub.com/pingcap/tidb/pkg/executor.(*ExecStmt).Exec.func1\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/executor/adapter.go:470\nruntime.gopanic\n\t/usr/local/go/src/runtime/panic.go:914\nruntime.goPanicIndex\n\t/usr/local/go/src/runtime/panic.go:114\ngithub.com/pingcap/tidb/pkg/table/tables.(*TableCommon).RemoveRecord\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/table/tables/tables.go:1376\ngithub.com/pingcap/tidb/pkg/executor.(*DeleteExec).removeRow\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/executor/delete.go:246\ngithub.com/pingcap/tidb/pkg/executor.(*DeleteExec).deleteOneRow\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/executor/delete.go:73\ngithub.com/pingcap/tidb/pkg/executor.(*DeleteExec).deleteSingleTableByChunk\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/executor/delete.go:140\ngithub.com/pingcap/tidb/pkg/executor.(*DeleteExec).Next\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/executor/delete.go:61\ngithub.com/pingcap/tidb/pkg/executor/internal/exec.Next\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/executor/internal/exec/executor.go:278\ngithub.com/pingcap/tidb/pkg/executor.(*ExecStmt).next\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/executor/adapter.go:1226\ngithub.com/pingcap/tidb/pkg/executor.(*ExecStmt).handleNoDelayExecutor\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/executor/adapter.go:971\ngithub.com/pingcap/tidb/pkg/executor.(*ExecStmt).handlePessimisticDML\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/executor/adapter.go:1032\ngithub.com/pingcap/tidb/pkg/executor.(*ExecStmt).handleNoDelay\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/executor/adapter.go:794\ngithub.com/pingcap/tidb/pkg/executor.(*ExecStmt).Exec\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/executor/adapter.go:578\ngithub.com/pingcap/tidb/pkg/session.runStmt\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/session/session.go:2421\ngithub.com/pingcap/tidb/pkg/session.(*session).ExecuteStmt\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/session/session.go:2271\ngithub.com/pingcap/tidb/pkg/server.(*TiDBContext).ExecuteStmt\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/server/driver_tidb.go:292\ngithub.com/pingcap/tidb/pkg/server.(*clientConn).executePreparedStmtAndWriteResult\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/server/conn_stmt.go:305\ngithub.com/pingcap/tidb/pkg/server.(*clientConn).executePlanCacheStmt\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/server/conn_stmt.go:233\ngithub.com/pingcap/tidb/pkg/server.(*clientConn).handleStmtExecute\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/server/conn_stmt.go:225\ngithub.com/pingcap/tidb/pkg/server.(*clientConn).dispatch\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/server/conn.go:1314\ngithub.com/pingcap/tidb/pkg/server.(*clientConn).Run\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/server/conn.go:1066\ngithub.com/pingcap/tidb/pkg/server.(*Server).onConn\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/server/server.go:701"]

[yibin87]

Reproduced again:
err="runtime error: index out of range [3] with length 3"] [stack="github.com/pingcap/tidb/pkg/server.(*clientConn).Run.func1\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/server/conn.go:968\nruntime.gopanic\n\t/usr/local/go/src/runtime/panic.go:914\ngithub.com/pingcap/tidb/pkg/executor.(*ExecStmt).Exec.func1\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/executor/adapter.go:470\nruntime.gopanic\n\t/usr/local/go/src/runtime/panic.go:914\nruntime.goPanicIndex\n\t/usr/local/go/src/runtime/panic.go:114\ngithub.com/pingcap/tidb/pkg/table/tables.(*TableCommon).RemoveRecord\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/table/tables/tables.go:1376\ngithub.com/pingcap/tidb/pkg/executor.(*DeleteExec).removeRow\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/executor/delete.go:246\ngithub.com/pingcap/tidb/pkg/executor.(*DeleteExec).deleteOneRow\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/executor/delete.go:73\ngithub.com/pingcap/tidb/pkg/executor.(*DeleteExec).deleteSingleTableByChunk\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/executor/delete.go:140\ngithub.com/pingcap/tidb/pkg/executor.(*DeleteExec).Next\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/executor/delete.go:61\ngithub.com/pingcap/tidb/pkg/executor/internal/exec.Next\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/executor/internal/exec/executor.go:278\ngithub.com/pingcap/tidb/pkg/executor.(*ExecStmt).next\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/executor/adapter.go:1226\ngithub.com/pingcap/tidb/pkg/executor.(*ExecStmt).handleNoDelayExecutor\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/executor/adapter.go:971\ngithub.com/pingcap/tidb/pkg/executor.(*ExecStmt).handlePessimisticDML\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/executor/adapter.go:1032\ngithub.com/pingcap/tidb/pkg/executor.(*ExecStmt).handleNoDelay\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/executor/adapter.go:794\ngithub.com/pingcap/tidb/pkg/executor.(*ExecStmt).Exec\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/executor/adapter.go:578\ngithub.com/pingcap/tidb/pkg/session.runStmt\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/session/session.go:2440\ngithub.com/pingcap/tidb/pkg/session.(*session).ExecuteStmt\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/session/session.go:2282\ngithub.com/pingcap/tidb/pkg/server.(*TiDBContext).ExecuteStmt\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/server/driver_tidb.go:292\ngithub.com/pingcap/tidb/pkg/server.(*clientConn).executePreparedStmtAndWriteResult\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/server/conn_stmt.go:305\ngithub.com/pingcap/tidb/pkg/server.(*clientConn).executePlanCacheStmt\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/server/conn_stmt.go:233\ngithub.com/pingcap/tidb/pkg/server.(*clientConn).handleStmtExecute\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/server/conn_stmt.go:225\ngithub.com/pingcap/tidb/pkg/server.(*clientConn).dispatch\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/server/conn.go:1314\ngithub.com/pingcap/tidb/pkg/server.(*clientConn).Run\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/server/conn.go:1066\ngithub.com/pingcap/tidb/pkg/server.(*Server).onConn\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/server/server.go:701"]
[2023/11/12 11:00:10.848 +08:00] [INFO] [coprocessor.go:1330] ["[TIME_COP_WAIT] resp_time:847.063407ms txnStartTS:445581363768721412 region_id:11488 store_addr:tc-tikv-2.tc-tikv-peer.endless-htap-ch-full-ap-query-tps-4320497-1-116.svc:20160 kv_process_ms:0 kv_wait_ms:0 kv_read_ms:0 processed_versions:2 total_versions:81 rocksdb_delete_skipped_count:60 rocksdb_key_skipped_count:140 rocksdb_cache_hit_count:11 rocksdb_read_count:0 rocksdb_read_byte:0"] [conn=3261075584] [session_alias=]
[2023/11/12 11:00:10.853 +08:00] [INFO] [coprocessor.go:1330] ["[TIME_COP_WAIT] resp_time:3.478962235s txnStartTS:445581363074564217 region_id:5567 store_addr:tc-tikv-2.tc-tikv-peer.endless-htap-ch-full-ap-query-tps-4320497-1-116.svc:20160 kv_process_ms:0 kv_wait_ms:0 kv_read_ms:0 processed_versions:2 total_versions:3 rocksdb_delete_skipped_count:0 rocksdb_key_skipped_count:2 rocksdb_cache_hit_count:11 rocksdb_read_count:0 rocksdb_read_byte:0

[you06]

@yibin87 seems it happens when deleting from a TPCC table. Is there any special setting when running the test? How can I reproduce it.

[zeminzhou]

Executing drop column while deleting the table maybe reproduce it.

[yibin87]

Same TCMS regression test, detect similar issue:
[conn.go:1013] ["connection running loop panic"] [conn=3789554060] [session_alias=] [lastSQL="UPDATE stock SET s_quantity = ?, s_ytd = s_ytd + ?, s_order_cnt = s_order_cnt + 1, s_remote_cnt = s_remote_cnt + ? WHERE s_i_id = ? AND s_w_id = ? [arguments: (40, 10, 0, 12823, 48)]"] [err="runtime error: index out of range [17] with length 17"] [stack="github.com/pingcap/tidb/pkg/server.(*clientConn).Run.func1\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/server/conn.go:1016\nruntime.gopanic\n\t/usr/local/go/src/runtime/panic.go:914\ngithub.com/pingcap/tidb/pkg/executor.(*Compiler).Compile.func1\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/executor/compiler.go:56\nruntime.gopanic\n\t/usr/local/go/src/runtime/panic.go:914\nruntime.goPanicIndex\n\t/usr/local/go/src/runtime/panic.go:114\ngithub.com/pingcap/tidb/pkg/planner/core.GetUpdateColumnsInfo\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/planner/core/logical_plan_builder.go:5857\ngithub.com/pingcap/tidb/pkg/planner/core.(*Update).buildTbl2UpdateColumns\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/planner/core/foreign_key.go:335\ngithub.com/pingcap/tidb/pkg/planner/core.(*Update).buildOnUpdateFKTriggers\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/planner/core/foreign_key.go:223\ngithub.com/pingcap/tidb/pkg/planner/core.buildPointUpdatePlan\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/planner/core/point_get_plan.go:1958\ngithub.com/pingcap/tidb/pkg/planner/core.tryUpdatePointPlan\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/planner/core/point_get_plan.go:1897\ngithub.com/pingcap/tidb/pkg/planner/core.TryFastPlan\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/planner/core/point_get_plan.go:911\ngithub.com/pingcap/tidb/pkg/planner.Optimize\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/planner/optimize.go:208\ngithub.com/pingcap/tidb/pkg/planner/core.generateNewPlan\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/planner/core/plan_cache.go:315\ngithub.com/pingcap/tidb/pkg/planner/core.GetPlanFromSessionPlanCache\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/planner/core/plan_cache.go:214\ngithub.com/pingcap/tidb/pkg/planner.OptimizeExecStmt\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/planner/optimize.go:525\ngithub.com/pingcap/tidb/pkg/planner.Optimize\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/planner/optimize.go:162\ngithub.com/pingcap/tidb/pkg/executor.(*Compiler).Compile\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/executor/compiler.go:104\ngithub.com/pingcap/tidb/pkg/session.(*session).ExecuteStmt\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/session/session.go:2197\ngithub.com/pingcap/tidb/pkg/server.(*TiDBContext).ExecuteStmt\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/server/driver_tidb.go:294\ngithub.com/pingcap/tidb/pkg/server.(*clientConn).executePreparedStmtAndWriteResult\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/server/conn_stmt.go:306\ngithub.com/pingcap/tidb/pkg/server.(*clientConn).executePlanCacheStmt\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/server/conn_stmt.go:234\ngithub.com/pingcap/tidb/pkg/server.(*clientConn).handleStmtExecute\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/server/conn_stmt.go:225\ngithub.com/pingcap/tidb/pkg/server.(*clientConn).dispatch\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/server/conn.go:1371\ngithub.com/pingcap/tidb/pkg/server.(*clientConn).Run\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/server/conn.go:1114\ngithub.com/pingcap/tidb/pkg/server.(*Server).onConn\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/pkg/server/server.go:737

[crazycs520]

#54255 already fixed this issue.