ccl/multiregionccl: TestDropRegionFailWithConcurrentBackupOps failed [too many concurrent timer firings]

[cockroach-teamcity]

ccl/multiregionccl.TestDropRegionFailWithConcurrentBackupOps failed with artifacts on master @ 7604dbd86f8fe417e97b67d3ff978850300e4f7a:

created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx in goroutine 325807
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:488 +0x37c

goroutine 321843 gp=0x40034aefc0 m=nil [select]:
runtime.gopark(0x40148ab658?, 0x2?, 0x50?, 0x70?, 0x40148ab614?)
	GOROOT/src/runtime/proc.go:424 +0xc8 fp=0x40173a8ca0 sp=0x40173a8c80 pc=0x489ab8
runtime.selectgo(0x40173a8e58, 0x40148ab610, 0x0?, 0x0, 0x401b55d798?, 0x1)
	GOROOT/src/runtime/select.go:335 +0x67c fp=0x40173a8dd0 sp=0x40173a8ca0 pc=0x464d4c
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0x40154f9008?, 0x7557af0?})
	github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:156 +0xb0 fp=0x40173a8eb0 sp=0x40173a8dd0 pc=0x3615290
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2({0x66a1d06?, 0x40148ab798?})
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:498 +0x17c fp=0x40173a8fb0 sp=0x40173a8eb0 pc=0x145416c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.gowrap1()
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:499 +0x34 fp=0x40173a8fd0 sp=0x40173a8fb0 pc=0x1453fb4
runtime.goexit({})
	src/runtime/asm_arm64.s:1223 +0x4 fp=0x40173a8fd0 sp=0x40173a8fd0 pc=0x492d04
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx in goroutine 321722
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:488 +0x37c

goroutine 293569 gp=0x40333a3dc0 m=nil [select]:
runtime.gopark(0x402a353e58?, 0x2?, 0xe8?, 0x3c?, 0x402a353e14?)
	GOROOT/src/runtime/proc.go:424 +0xc8 fp=0x400c50bca0 sp=0x400c50bc80 pc=0x489ab8
runtime.selectgo(0x400c50be58, 0x402a353e10, 0x4042a10738?, 0x0, 0x1?, 0x1)
	GOROOT/src/runtime/select.go:335 +0x67c fp=0x400c50bdd0 sp=0x400c50bca0 pc=0x464d4c
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0x4005ddf808?, 0x7557af0?})
	github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:156 +0xb0 fp=0x400c50beb0 sp=0x400c50bdd0 pc=0x3615290
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2({0x66a1d06?, 0x4012a92fb0?})
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:498 +0x17c fp=0x400c50bfb0 sp=0x400c50beb0 pc=0x145416c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.gowrap1()
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:499 +0x34 fp=0x400c50bfd0 sp=0x400c50bfb0 pc=0x1453fb4
runtime.goexit({})
	src/runtime/asm_arm64.s:1223 +0x4 fp=0x400c50bfd0 sp=0x400c50bfd0 pc=0x492d04
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx in goroutine 293023
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:488 +0x37c

goroutine 298606 gp=0x40346d0380 m=nil [select]:
runtime.gopark(0x4016affe88?, 0x2?, 0xa0?, 0xf7?, 0x4016affe64?)
	GOROOT/src/runtime/proc.go:424 +0xc8 fp=0x400ce0bcf0 sp=0x400ce0bcd0 pc=0x489ab8
runtime.selectgo(0x400ce0be88, 0x4016affe60, 0x0?, 0x0, 0x40242f5e88?, 0x1)
	GOROOT/src/runtime/select.go:335 +0x67c fp=0x400ce0be20 sp=0x400ce0bcf0 pc=0x464d4c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Start.func2({0x40241c5808?, 0x7557af0?})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:2325 +0xac fp=0x400ce0beb0 sp=0x400ce0be20 pc=0x3a7604c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2({0x66a8fec?, 0x0?})
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:498 +0x17c fp=0x400ce0bfb0 sp=0x400ce0beb0 pc=0x145416c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.gowrap1()
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:499 +0x34 fp=0x400ce0bfd0 sp=0x400ce0bfb0 pc=0x1453fb4
runtime.goexit({})
	src/runtime/asm_arm64.s:1223 +0x4 fp=0x400ce0bfd0 sp=0x400ce0bfd0 pc=0x492d04
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx in goroutine 298518
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:488 +0x37c

Help

See also: How To Investigate a Go Test Failure (internal)

/cc @cockroachdb/sql-foundations

This test on roachdash | Improve this report!

Jira issue: CRDB-43164

[annrpom]

fatal error: too many concurrent timer firings
      
      runtime stack:
      runtime.throw({0x6736827?, 0x0?})
        GOROOT/src/runtime/panic.go:1067 +0x38 fp=0xffff627fdaa0 sp=0xffff627fda70 pc=0x489998
      runtime.(*timer).unlockAndRun(0x40001982b0, 0x0?)
        GOROOT/src/runtime/time.go:1076 +0x2e0 fp=0xffff627fdb10 sp=0xffff627fdaa0 pc=0x471ab0
      runtime.(*timers).run(0x40000da790, 0x1aa4d6d146d)
        GOROOT/src/runtime/time.go:1008 +0x124 fp=0xffff627fdb40 sp=0xffff627fdb10 pc=0x471774
      runtime.(*timers).check(0x40000da790, 0x0?)
        GOROOT/src/runtime/time.go:942 +0x170 fp=0xffff627fdb90 sp=0xffff627fdb40 pc=0x471570
      runtime.stealWork(0x0?)
        GOROOT/src/runtime/proc.go:3707 +0x220 fp=0xffff627fdc00 sp=0xffff627fdb90 pc=0x458cd0
      runtime.findRunnable()
        GOROOT/src/runtime/proc.go:3384 +0x46c fp=0xffff627fdd70 sp=0xffff627fdc00 pc=0x457a2c
      runtime.schedule()
        GOROOT/src/runtime/proc.go:4015 +0x98 fp=0xffff627fddb0 sp=0xffff627fdd70 pc=0x459988
      runtime.park_m(0x4005883c00)
        GOROOT/src/runtime/proc.go:4130 +0x244 fp=0xffff627fde10 sp=0xffff627fddb0 pc=0x459e34
      runtime.mcall()
        src/runtime/asm_arm64.s:193 +0x54 fp=0xffff627fde20 sp=0xffff627fde10 pc=0x4907c4

[annrpom]

similar to #132578 (comment)

[rafiss]

#132596 has been closed