Concurrent issue when sending BLS-sig tx in a gorouting

[gitferry]

A rare bug appeared when running integration tests in CI. The cause is that when sending BLS-sig tx in a goroutine, the cosmos-sdk's IAVL tree is accessed concurrently. The error stack is as follows:

fatal error: concurrent map iteration and map write

goroutine 111849 [running]:
runtime.throw({0x1d84ac7?, 0x1d?})
	/usr/local/go/src/runtime/panic.go:992 +0x71 fp=0xc0045243a0 sp=0xc004524370 pc=0x4401d1
runtime.mapiternext(0x1cd9d40?)
	/usr/local/go/src/runtime/map.go:871 +0x4eb fp=0xc004524410 sp=0xc0045243a0 pc=0x417a2b
runtime.mapiterinit(0xc001404da0?, 0x9?, 0x9?)
	/usr/local/go/src/runtime/map.go:861 +0x228 fp=0xc004524430 sp=0xc004524410 pc=0x4174e8
github.com/cosmos/iavl.NewUnsavedFastIterator({0xc001404da0, 0x9, 0x9}, {0xc001404db0, 0x9, 0x9}, 0x1, 0xc001241100, 0xc00276e240, 0xc00276e2a0)
	/home/circleci/.go_workspace/pkg/mod/github.com/cosmos/iavl@v0.19.3/unsaved_fast_iterator.go:55 +0x227 fp=0xc004524520 sp=0xc004524430 pc=0xe51607
github.com/cosmos/iavl.(*MutableTree).Iterator(0xc00367f400, {0xc001404da0, 0x9, 0x9}, {0xc001404db0, 0x9, 0x9}, 0x38?)
	/home/circleci/.go_workspace/pkg/mod/github.com/cosmos/iavl@v0.19.3/mutable_tree.go:216 +0x10c fp=0xc004524588 sp=0xc004524520 pc=0xe3830c
github.com/cosmos/cosmos-sdk/store/iavl.(*Store).Iterator(0x30?, {0xc001404da0?, 0x0?, 0x273a738?}, {0xc001404db0?, 0x18?, 0x41b8e5?})
	/home/circleci/.go_workspace/pkg/mod/github.com/cosmos/cosmos-sdk@v0.46.3/store/iavl/store.go:245 +0x31 fp=0xc0045245d8 sp=0xc004524588 pc=0xe6b791
github.com/cosmos/cosmos-sdk/store/cachekv.(*Store).iterator(0xc00330c780, {0xc001404da0, 0x9, 0x9}, {0xc001404db0, 0x9, 0x9}, 0x1)
	/home/circleci/.go_workspace/pkg/mod/github.com/cosmos/cosmos-sdk@v0.46.3/store/cachekv/store.go:186 +0x126 fp=0xc004524678 sp=0xc0045245d8 pc=0xe25f46
github.com/cosmos/cosmos-sdk/store/cachekv.(*Store).Iterator(0x0?, {0xc001404da0?, 0x0?, 0x0?}, {0xc001404db0?, 0x0?, 0x415025?})
	/home/circleci/.go_workspace/pkg/mod/github.com/cosmos/cosmos-sdk@v0.46.3/store/cachekv/store.go:171 +0x29 fp=0xc0045246c8 sp=0xc004524678 pc=0xe25d29
github.com/cosmos/cosmos-sdk/store/gaskv.(*Store).iterator(0xc001887d40, {0xc001404da0?, 0x0?, 0x0?}, {0xc001404db0?, 0x1?, 0x1?}, 0xb0?)
	/home/circleci/.go_workspace/pkg/mod/github.com/cosmos/cosmos-sdk@v0.46.3/store/gaskv/store.go:103 +0x56 fp=0xc004524770 sp=0xc0045246c8 pc=0xc87916
github.com/cosmos/cosmos-sdk/store/gaskv.(*Store).Iterator(0x0?, {0xc001404da0?, 0x0?, 0x3501f50?}, {0xc001404db0?, 0x0?, 0x35d5958?})
	/home/circleci/.go_workspace/pkg/mod/github.com/cosmos/cosmos-sdk@v0.46.3/store/gaskv/store.go:74 +0x29 fp=0xc0045247c0 sp=0xc004524770 pc=0xc87709
github.com/cosmos/cosmos-sdk/store/prefix.Store.Iterator({{0x2743ac0, 0xc001887d40}, {0x3501f50, 0x1, 0x1}}, {0xc001404d90, 0x8, 0x8}, {0xc001404d98, 0x8, ...})
	/home/circleci/.go_workspace/pkg/mod/github.com/cosmos/cosmos-sdk@v0.46.3/store/prefix/store.go:101 +0x239 fp=0xc004524868 sp=0xc0045247c0 pc=0xed8939
github.com/cosmos/cosmos-sdk/store/prefix.(*Store).Iterator(0x8?, {0xc001404d90?, 0x0?, 0xc001404d88?}, {0xc001404d98?, 0xc001887d40?, 0x3501f50?})
	<autogenerated>:1 +0xa5 fp=0xc0045248f8 sp=0xc004524868 pc=0xeda1a5
github.com/cosmos/cosmos-sdk/store/prefix.Store.Iterator({{0x27440d8, 0xc002350f30}, {0xc001404d88, 0x8, 0x8}}, {0x0, 0x0, 0x0}, {0x0, 0x0, ...})
	/home/circleci/.go_workspace/pkg/mod/github.com/cosmos/cosmos-sdk@v0.46.3/store/prefix/store.go:101 +0x239 fp=0xc0045249a0 sp=0xc0045248f8 pc=0xed8939
github.com/babylonchain/babylon/x/epoching/keeper.Keeper.GetValidatorSet({{0x27442e0, 0xc002ac7410}, {0x2725fc8, 0xc0000f47b0}, {0x2725fc8, 0xc0000f47b0}, {0x0, 0x0}, {{0x27442e0, 0xc002ac7410}, ...}, ...}, ...)
	/home/circleci/project/x/epoching/keeper/epoch_val_set.go:16 +0x137 fp=0xc004524d78 sp=0xc0045249a0 pc=0xfc59d7
github.com/babylonchain/babylon/x/epoching/keeper.(*Keeper).GetValidatorSet(_, {{0x2739fc8, 0xc000138000}, {0x27457a0, 0xc00330cc40}, {{0x0, 0x0}, {0x0, 0x0}, 0x204, ...}, ...}, ...)
	<autogenerated>:1 +0xa5 fp=0xc004525120 sp=0xc004524d78 pc=0xfd6d65
github.com/babylonchain/babylon/x/checkpointing/keeper.Keeper.GetValidatorSet(...)
	/home/circleci/project/x/checkpointing/keeper/keeper.go:321
github.com/babylonchain/babylon/x/checkpointing/keeper.Keeper.SendBlsSig({{0x27442e0, 0xc002ac7410}, {0x2725fc8, 0xc0000f47e0}, {0x2725fc8, 0x0}, {0x27285a0, 0xc002bd22a0}, {0x273cb90, 0xc0022f8a50}, ...}, ...)
	/home/circleci/project/x/checkpointing/keeper/bls_signer.go:23 +0xbf fp=0xc004525b28 sp=0xc004525120 pc=0x174801f
github.com/babylonchain/babylon/x/checkpointing.BeginBlocker.func1()
	/home/circleci/project/x/checkpointing/abci.go:45 +0x9f fp=0xc004525fe0 sp=0xc004525b28 pc=0x1756aff
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1571 +0x1 fp=0xc004525fe8 sp=0xc004525fe0 pc=0x473681
created by github.com/babylonchain/babylon/x/checkpointing.BeginBlocker
	/home/circleci/project/x/checkpointing/abci.go:44 +0x51b

goroutine 1 [chan receive]:
testing.runFuzzTests({0x2746088?, 0x35d5958}, {0x34f3fe0, 0xa, 0x7f930526fa68?}, {0x40?, 0x359f2c0?, 0x359f2c0?})
	/usr/local/go/src/testing/fuzz.go:521 +0x754
testing.(*M).Run(0xc0000fe0a0)
	/usr/local/go/src/testing/testing.go:1720 +0x61f
main.main()
	_testmain.go:79 +0x1aa

[gitferry]

Resolved by #202