ccl/backupccl: TestDataDriven_restore_regionless_mixed_version failed [too many concurrent timer firings]

[cockroach-teamcity]

ccl/backupccl.TestDataDriven_restore_regionless_mixed_version failed with artifacts on master @ a642b9199f8e470d8cc7ba2253c9d42caab3fe4b:

github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0x400c70fc20?)
	github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:147 +0x54 fp=0x40101e72c0 sp=0x40101e7280 pc=0x10eeb64
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).Start(0x400c70fc20, {0x76958d8?, 0x4011bba280?})
	github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:110 +0x68 fp=0x40101e7330 sp=0x40101e72c0 pc=0x31a50d8
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBaseNoHelper).Run(0x400c70fc20, {0x76958d8?, 0x4011bba280?}, {0x7655688, 0x4004cfc808})
	github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:726 +0x4c fp=0x40101e7370 sp=0x40101e7330 pc=0x249f60c
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).Run(0x76958d8?, {0x76958d8?, 0x4011bba280?}, {0x7655688?, 0x4004cfc808?})
	<autogenerated>:1 +0x38 fp=0x40101e73b0 sp=0x40101e7370 pc=0x31b2068
github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0x40072b9188, {0x76958d8, 0x4011bba280}, 0x10?)
	github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:574 +0x17c fp=0x40101e7480 sp=0x40101e73b0 pc=0x287055c
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0x771c7c0?, {0x76958d8?, 0x4011bba280?}, 0x20?)
	github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:300 +0x1f0 fp=0x40101e7530 sp=0x40101e7480 pc=0x31aa500
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0x40078df340, {0x7695910, 0x400c1d7890}, 0x4003f0d6c0, 0x4007dd9c20, 0x4001797280, 0x4004cfc808, 0x400d7e95b8, 0x40101e7c90)
	github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:923 +0xa04 fp=0x40101e7ae0 sp=0x40101e7530 pc=0x363d964
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x40078df340, {0x7695910, 0x400c1d7890}, 0x400d7e95b8, 0x4003f0d6c0, 0x4007dd9c20, {{0x7696198, 0x400ef1e660}, 0x0}, 0x4004cfc808, ...)
	github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1985 +0x174 fp=0x40101e7c10 sp=0x40101e7ae0 pc=0x36421a4
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRunAll.func3(0x0?, 0x0?, {0x7695910?, 0x400c1d7890?}, 0xffff31690de0?, 0x10?, 0x400f091d18?)
	github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1699 +0x98 fp=0x40101e7cb0 sp=0x40101e7c10 pc=0x3640668
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRunAll(0x40078df340, {0x7695910, 0x400c1d7890}, 0x400d7e95b8, 0x4003f0d6c0, 0x400d7e90b0, 0x4004cfc808, 0x0)
	github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1702 +0x164 fp=0x40101e7e00 sp=0x40101e7cb0 pc=0x36400a4
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0x400d7e8a08, {0x7695910, 0x400c1d7890}, 0x400d7e90b0, 0x0?, {0x771f130?, 0x4001665630?}, 0x0, 0x40067f1598, {0x0, ...})
	github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:2433 +0x358 fp=0x40101e7f00 sp=0x40101e7e00 pc=0x35bac38
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0x400d7e8a08, {0x76958d8, 0x40016657c0}, 0x400d7e90b0, {0x771f130, 0x4001665630})
	github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1980 +0xd28 fp=0x40101e83c0 sp=0x40101e7f00 pc=0x35b8218
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0x400d7e8a08, {0x7695910, 0x400c1d6780}, {{0x76bea00, 0x400b2d7e80}, {0x0, 0x0, 0x0}, {0x6a5220c, 0x13d}, ...}, ...)
	github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1187 +0x3e70 fp=0x40101e9050 sp=0x40101e83c0 pc=0x35b1180
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt.func1({0x7695910?, 0x400c1d6780?})
	github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:141 +0x8c fp=0x40101e9100 sp=0x40101e9050 pc=0x35acaac
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithProfiling(0x7695910?, {0x7695910?, 0x400c1d6780?}, {0x76bea00?, 0x400b2d7e80?}, 0x426cf4?, 0x2?)
	github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:3449 +0x25c fp=0x40101e9250 sp=0x40101e9100 pc=0x35c0f8c
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0x400d7e8a08, {0x7695910, 0x400c1d6780}, {{0x76bea00, 0x400b2d7e80}, {0x0, 0x0, 0x0}, {0x6a5220c, 0x13d}, ...}, ...)
	github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:140 +0x544 fp=0x40101e9430 sp=0x40101e9250 pc=0x35ac504
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0x400d7e8a08, {0x7695910, 0x400c1d6780}, {{0x0, 0x0}, 0x4007f5c6e0, {0x400e5d4e60, 0x2, 0x2}, {0x0, ...}, ...}, ...)
	github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:246 +0x35c fp=0x40101e95d0 sp=0x40101e9430 pc=0x35aceac
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0x0, 0x0}, 0x0, {0xc1bbc9e801b7cf4d, 0x1e64e4f1e, 0x0}, 0x1}, 0x400d7e8a08, 0x40101e9ba0, 0x40101e9b90, ...)
	github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2446 +0x7f8 fp=0x40101e9850 sp=0x40101e95d0 pc=0x359da68
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0x400d7e8a08)
	github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2448 +0x5bc fp=0x40101e9d20 sp=0x40101e9850 pc=0x359bc5c
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0x400d7e8a08, {0x7695910, 0x400c1d67b0}, 0xc2c4e0?, 0x40053f8648?, 0xc2adc4?)
	github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2263 +0x168 fp=0x40101e9e00 sp=0x40101e9d20 pc=0x359b338
github.com/cockroachdb/cockroach/pkg/sql.(*InternalExecutor).runWithEx.func2({0x7695910, 0x400c1d67b0})
	github.com/cockroachdb/cockroach/pkg/sql/internal.go:251 +0xb8 fp=0x40101e9eb0 sp=0x40101e9e00 pc=0x36991c8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2({0x67922d8?, 0x40053f87b0?})
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:498 +0x17c fp=0x40101e9fb0 sp=0x40101e9eb0 pc=0x12741cc
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.gowrap1()
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:499 +0x34 fp=0x40101e9fd0 sp=0x40101e9fb0 pc=0x1274014
runtime.goexit({})
	src/runtime/asm_arm64.s:1223 +0x4 fp=0x40101e9fd0 sp=0x40101e9fd0 pc=0x492994
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx in goroutine 17272
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:488 +0x37c

Help

See also: How To Investigate a Go Test Failure (internal)

/cc @cockroachdb/disaster-recovery

This test on roachdash | Improve this report!

Jira issue: CRDB-43224

[annrpom]

fatal error: too many concurrent timer firings
      
      runtime stack:
      runtime.throw({0x6826d2b?, 0x0?})
        GOROOT/src/runtime/panic.go:1067 +0x38 fp=0xffff2306caa0 sp=0xffff2306ca70 pc=0x489628
      runtime.(*timer).unlockAndRun(0x4000ea8010, 0x0?)

[annrpom]

Dupe of #132596