Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

shutdown deadlock #12796

Open
AskAlexSharov opened this issue Nov 20, 2024 · 0 comments
Open

shutdown deadlock #12796

AskAlexSharov opened this issue Nov 20, 2024 · 0 comments
Labels
imp2 Medium importance

Comments

@AskAlexSharov
Copy link
Collaborator

gnosis, main:

[DBUG] [11-20|08:03:56.031] [6/6 Finish] DONE                        in=33.888µs
[DBUG] [11-20|08:03:56.033] RPC Daemon notified of new headers       from=37116117 to=37116119 amount=1 header sending=63.916µs
[DBUG] [11-20|08:03:56.035] [txpool] New block                       block=37116118 unwound=0 mined=0 baseFee=7 pending-pre=5 pending=5 baseFee=0 queued=360 err=nil
[INFO] [11-20|08:03:56.036] head updated                             age=1s head=0xa88a33c9ab88a0e6d286a3945ebae132777daa6930d4bdd8c4e547460d63fc2a hash=0xa88a33c9ab88a0e6d286a3945ebae132777daa6930d4bdd8c4e547460d63fc2a number=37116118 execution=22.8055ms mgas/s=0.00 average mgas/s=7.06 commit=835.506µs alloc=6.5GB sys=12.2GB
[DBUG] [11-20|08:03:56.037] [6/6 Finish] Prune done                  in=19.601µs
[DBUG] [11-20|08:03:56.056] [5/6 TxLookup] Prune done                in=18.687651ms
[DBUG] [11-20|08:03:56.057] [4/6 Execution] Prune done               in=881.745µs
[DBUG] [11-20|08:03:56.057] [3/6 Senders] Prune done                 in=29.56µs
[DBUG] [11-20|08:03:56.057] [2/6 BlockHashes] Prune done             in=26.569µs
[DBUG] [11-20|08:03:56.057] [snapshots] Prune Blocks                 to=37114976 limit=10
[DBUG] [11-20|08:03:56.060] [1/6 OtterSync] Prune done               in=2.340166ms
[DBUG] [11-20|08:03:56.622] Post forkchoice operations               app=caplin stage=ForkChoice duration=1.37141ms
[DBUG] [11-20|08:03:56.623] Imported chain segment                   app=caplin stage=ForkChoice hash=0x768501a5110293405d695857925f6a75e03416fcac9dfc14e2a752a44f71a49e slot=18619299 alloc=6.6GB sys=12.2GB
[DBUG] [11-20|08:03:56.624] clstage finish                           app=caplin stage=ForkChoice in=678.386319ms next=SleepForSlot
[DBUG] [11-20|08:03:56.769] import operations                        time=767.287557ms
SIGABRT: abort
PC=0x4a5f41 m=0 sigcode=0

goroutine 0 gp=0x7c1fd00 m=0 mp=0x7c21e60 [idle]:
runtime.futex(0x7c21fa0, 0x80, 0x0, 0x0, 0x0, 0x0)
	/usr/local/go/src/runtime/sys_linux_amd64.s:557 +0x21 fp=0x7fff9e90e0e0 sp=0x7fff9e90e0d8 pc=0x4a5f41
runtime.futexsleep(0x7fff9e90e158?, 0x434570?, 0x0?)
	/usr/local/go/src/runtime/os_linux.go:69 +0x30 fp=0x7fff9e90e130 sp=0x7fff9e90e0e0 pc=0x461bf0
runtime.notesleep(0x7c21fa0)
	/usr/local/go/src/runtime/lock_futex.go:170 +0x87 fp=0x7fff9e90e168 sp=0x7fff9e90e130 pc=0x434707
runtime.mPark(...)
	/usr/local/go/src/runtime/proc.go:1761
runtime.stopm()
	/usr/local/go/src/runtime/proc.go:2782 +0x8c fp=0x7fff9e90e198 sp=0x7fff9e90e168 pc=0x46d72c
runtime.exitsyscall0(0xc1747b2e00)
	/usr/local/go/src/runtime/proc.go:4755 +0x205 fp=0x7fff9e90e1e0 sp=0x7fff9e90e198 pc=0x4724a5
runtime.mcall()
	/usr/local/go/src/runtime/asm_amd64.s:458 +0x4e fp=0x7fff9e90e1f8 sp=0x7fff9e90e1e0 pc=0x4a19ae

goroutine 1 gp=0xc0000061c0 m=nil [chan receive, 641 minutes]:
runtime.gopark(0x472680?, 0xc0323e0690?, 0xc0?, 0x61?, 0x4a6d29?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc0375593c0 sp=0xc0375593a0 pc=0x46904e
runtime.chanrecv(0xc000110240, 0x0, 0x1)
	/usr/local/go/src/runtime/chan.go:583 +0x36d fp=0xc037559438 sp=0xc0375593c0 pc=0x42e42d
runtime.chanrecv1(0xc0005fcb60?, 0x4766485?)
	/usr/local/go/src/runtime/chan.go:442 +0x12 fp=0xc037559460 sp=0xc037559438 pc=0x42e092
github.com/erigontech/erigon/node.(*Node).Wait(...)
	/home/ubuntu/erigon/node/node.go:274
github.com/erigontech/erigon/turbo/node.(*ErigonNode).Serve(0xc037481530)
	/home/ubuntu/erigon/turbo/node/node.go:51 +0xd7 fp=0xc0375594e8 sp=0xc037559460 pc=0x3e46cb7
main.runErigon(0xc0009a8400)
	/home/ubuntu/erigon/cmd/erigon/main.go:106 +0x686 fp=0xc0375597f8 sp=0xc0375594e8 pc=0x3e86d66
github.com/erigontech/erigon/turbo/app.MakeApp.func1(0xc0009a8400)
	/home/ubuntu/erigon/turbo/app/make_app.go:71 +0x17f fp=0xc037559978 sp=0xc0375597f8 pc=0x3e65a9f
github.com/urfave/cli/v2.(*Command).Run(0xc0001c06e0, 0xc0009a8400, {0xc000142000, 0x15, 0x16})
	/home/ubuntu/go/pkg/mod/github.com/urfave/cli/v2@v2.27.5/command.go:276 +0x1579 fp=0xc037559d18 sp=0xc037559978 pc=0xa54919
github.com/urfave/cli/v2.(*App).RunContext(0xc0005d9800, {0x5238b80, 0x8bc5de0}, {0xc000142000, 0x15, 0x16})
	/home/ubuntu/go/pkg/mod/github.com/urfave/cli/v2@v2.27.5/app.go:333 +0x1295 fp=0xc037559e78 sp=0xc037559d18 pc=0xa4db95
github.com/urfave/cli/v2.(*App).Run(...)
	/home/ubuntu/go/pkg/mod/github.com/urfave/cli/v2@v2.27.5/app.go:307
main.main()
	/home/ubuntu/erigon/cmd/erigon/main.go:51 +0xc9 fp=0xc037559f50 sp=0xc037559e78 pc=0x3e86509
runtime.main()
	/usr/local/go/src/runtime/proc.go:271 +0x29d fp=0xc037559fe0 sp=0xc037559f50 pc=0x468bdd
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc037559fe8 sp=0xc037559fe0 pc=0x4a39e1

goroutine 2 gp=0xc000006c40 m=nil [force gc (idle), 2 minutes]:
runtime.gopark(0x1b90c49c3f5a8e?, 0x7c21e60?, 0x0?, 0x0?, 0x0?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc0c7c36fa8 sp=0xc0c7c36f88 pc=0x46904e
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:408
runtime.forcegchelper()
	/usr/local/go/src/runtime/proc.go:326 +0xb3 fp=0xc0c7c36fe0 sp=0xc0c7c36fa8 pc=0x468eb3
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc0c7c36fe8 sp=0xc0c7c36fe0 pc=0x4a39e1
created by runtime.init.6 in goroutine 1
	/usr/local/go/src/runtime/proc.go:314 +0x1a

goroutine 3 gp=0xc000007180 m=nil [GC sweep wait]:
runtime.gopark(0x7c33c01?, 0x710100000400?, 0x0?, 0x0?, 0x0?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc0000a4f50 sp=0xc0000a4f30 pc=0x46904e
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:408
runtime.bgsweep(0xc000056150)
	/usr/local/go/src/runtime/mgcsweep.go:318 +0xdf fp=0xc0000a4fc8 sp=0xc0000a4f50 pc=0x4512bf
runtime.gcenable.gowrap1()
	/usr/local/go/src/runtime/mgc.go:203 +0x25 fp=0xc0000a4fe0 sp=0xc0000a4fc8 pc=0x444f65
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc0000a4fe8 sp=0xc0000a4fe0 pc=0x4a39e1
created by runtime.gcenable in goroutine 1
	/usr/local/go/src/runtime/mgc.go:203 +0x66

goroutine 4 gp=0xc000007340 m=nil [GC scavenge wait]:
runtime.gopark(0x67b7b8?, 0x659ac6?, 0x0?, 0x0?, 0x0?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc00009ef78 sp=0xc00009ef58 pc=0x46904e
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:408
runtime.(*scavengerState).park(0x7c1ebe0)
	/usr/local/go/src/runtime/mgcscavenge.go:425 +0x49 fp=0xc00009efa8 sp=0xc00009ef78 pc=0x44ec69
runtime.bgscavenge(0xc000056150)
	/usr/local/go/src/runtime/mgcscavenge.go:658 +0x59 fp=0xc00009efc8 sp=0xc00009efa8 pc=0x44f1f9
runtime.gcenable.gowrap2()
	/usr/local/go/src/runtime/mgc.go:204 +0x25 fp=0xc00009efe0 sp=0xc00009efc8 pc=0x444f05
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc00009efe8 sp=0xc00009efe0 pc=0x4a39e1
created by runtime.gcenable in goroutine 1
	/usr/local/go/src/runtime/mgc.go:204 +0xa5

goroutine 18 gp=0xc000102700 m=nil [finalizer wait]:
runtime.gopark(0x0?, 0x4c8d2d0?, 0x0?, 0xc0?, 0x2000000020?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc000184e20 sp=0xc000184e00 pc=0x46904e
runtime.runfinq()
	/usr/local/go/src/runtime/mfinal.go:194 +0x145 fp=0xc000184fe0 sp=0xc000184e20 pc=0x443f25
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc000184fe8 sp=0xc000184fe0 pc=0x4a39e1
created by runtime.createfing in goroutine 1
	/usr/local/go/src/runtime/mfinal.go:164 +0x3d

goroutine 19 gp=0xc0001028c0 m=nil [GC worker (idle), 4 minutes]:
runtime.gopark(0x1b908c9cd0c7c8?, 0x3?, 0x63?, 0x20?, 0x0?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc000183f50 sp=0xc000183f30 pc=0x46904e
runtime.gcBgMarkWorker()
	/usr/local/go/src/runtime/mgc.go:1310 +0xe5 fp=0xc000183fe0 sp=0xc000183f50 pc=0x447045
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc000183fe8 sp=0xc000183fe0 pc=0x4a39e1
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/local/go/src/runtime/mgc.go:1234 +0x1c

goroutine 34 gp=0xc00018a000 m=nil [GC worker (idle)]:
runtime.gopark(0x1b90c4bbff2bd8?, 0x1?, 0x8b?, 0x80?, 0x0?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc000182f50 sp=0xc000182f30 pc=0x46904e
runtime.gcBgMarkWorker()
	/usr/local/go/src/runtime/mgc.go:1310 +0xe5 fp=0xc000182fe0 sp=0xc000182f50 pc=0x447045
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc000182fe8 sp=0xc000182fe0 pc=0x4a39e1
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/local/go/src/runtime/mgc.go:1234 +0x1c

goroutine 5 gp=0xc000007880 m=nil [GC worker (idle)]:
runtime.gopark(0x1b90c4bbff33a2?, 0x1?, 0x34?, 0xb2?, 0x0?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc000181f50 sp=0xc000181f30 pc=0x46904e
runtime.gcBgMarkWorker()
	/usr/local/go/src/runtime/mgc.go:1310 +0xe5 fp=0xc000181fe0 sp=0xc000181f50 pc=0x447045
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc000181fe8 sp=0xc000181fe0 pc=0x4a39e1
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/local/go/src/runtime/mgc.go:1234 +0x1c

goroutine 20 gp=0xc000102a80 m=nil [GC worker (idle), 4 minutes]:
runtime.gopark(0x1b908c9cd0c5c1?, 0x3?, 0xb7?, 0x7?, 0x0?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc000180f50 sp=0xc000180f30 pc=0x46904e
runtime.gcBgMarkWorker()
	/usr/local/go/src/runtime/mgc.go:1310 +0xe5 fp=0xc000180fe0 sp=0xc000180f50 pc=0x447045
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc000180fe8 sp=0xc000180fe0 pc=0x4a39e1
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/local/go/src/runtime/mgc.go:1234 +0x1c

goroutine 35 gp=0xc00018a1c0 m=nil [GC worker (idle), 4 minutes]:
runtime.gopark(0x1b908c9cd0c8c5?, 0x3?, 0x59?, 0x63?, 0x0?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc00019bf50 sp=0xc00019bf30 pc=0x46904e
runtime.gcBgMarkWorker()
	/usr/local/go/src/runtime/mgc.go:1310 +0xe5 fp=0xc00019bfe0 sp=0xc00019bf50 pc=0x447045
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc00019bfe8 sp=0xc00019bfe0 pc=0x4a39e1
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/local/go/src/runtime/mgc.go:1234 +0x1c

goroutine 36 gp=0xc00018a380 m=nil [GC worker (idle), 93 minutes]:
runtime.gopark(0x1b8bb56ea0ce61?, 0x3?, 0x42?, 0xba?, 0x0?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc00019af50 sp=0xc00019af30 pc=0x46904e
runtime.gcBgMarkWorker()
	/usr/local/go/src/runtime/mgc.go:1310 +0xe5 fp=0xc00019afe0 sp=0xc00019af50 pc=0x447045
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc00019afe8 sp=0xc00019afe0 pc=0x4a39e1
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/local/go/src/runtime/mgc.go:1234 +0x1c

goroutine 37 gp=0xc00018a540 m=nil [GC worker (idle), 4 minutes]:
runtime.gopark(0x1b908c9cd03afc?, 0x1?, 0x27?, 0x96?, 0x0?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc000199f50 sp=0xc000199f30 pc=0x46904e
runtime.gcBgMarkWorker()
	/usr/local/go/src/runtime/mgc.go:1310 +0xe5 fp=0xc000199fe0 sp=0xc000199f50 pc=0x447045
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc000199fe8 sp=0xc000199fe0 pc=0x4a39e1
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/local/go/src/runtime/mgc.go:1234 +0x1c

goroutine 38 gp=0xc00018a700 m=nil [GC worker (idle), 4 minutes]:
runtime.gopark(0x8bcc1a0?, 0x3?, 0x29?, 0xf2?, 0x0?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc000198f50 sp=0xc000198f30 pc=0x46904e
runtime.gcBgMarkWorker()
	/usr/local/go/src/runtime/mgc.go:1310 +0xe5 fp=0xc000198fe0 sp=0xc000198f50 pc=0x447045
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc000198fe8 sp=0xc000198fe0 pc=0x4a39e1
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/local/go/src/runtime/mgc.go:1234 +0x1c

goroutine 6 gp=0xc000007a40 m=nil [GC worker (idle), 4 minutes]:
runtime.gopark(0x1b908c9cd0c819?, 0x1?, 0x18?, 0xbb?, 0x0?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc000197f50 sp=0xc000197f30 pc=0x46904e
runtime.gcBgMarkWorker()
	/usr/local/go/src/runtime/mgc.go:1310 +0xe5 fp=0xc000197fe0 sp=0xc000197f50 pc=0x447045
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc000197fe8 sp=0xc000197fe0 pc=0x4a39e1
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/local/go/src/runtime/mgc.go:1234 +0x1c

goroutine 21 gp=0xc000102c40 m=nil [GC worker (idle)]:
runtime.gopark(0x1b90c4bbff2b91?, 0x1?, 0x31?, 0x56?, 0x0?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc000196f50 sp=0xc000196f30 pc=0x46904e
runtime.gcBgMarkWorker()
	/usr/local/go/src/runtime/mgc.go:1310 +0xe5 fp=0xc000196fe0 sp=0xc000196f50 pc=0x447045
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc000196fe8 sp=0xc000196fe0 pc=0x4a39e1
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/local/go/src/runtime/mgc.go:1234 +0x1c

goroutine 39 gp=0xc00018a8c0 m=nil [GC worker (idle), 4 minutes]:
runtime.gopark(0x1b908c9cd03bb2?, 0x3?, 0x5c?, 0x2a?, 0x0?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc000195f50 sp=0xc000195f30 pc=0x46904e
runtime.gcBgMarkWorker()
	/usr/local/go/src/runtime/mgc.go:1310 +0xe5 fp=0xc000195fe0 sp=0xc000195f50 pc=0x447045
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc000195fe8 sp=0xc000195fe0 pc=0x4a39e1
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/local/go/src/runtime/mgc.go:1234 +0x1c

goroutine 40 gp=0xc00018aa80 m=nil [GC worker (idle), 4 minutes]:
runtime.gopark(0x1b908c9cd03ba5?, 0x3?, 0x58?, 0x17?, 0x0?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc000194f50 sp=0xc000194f30 pc=0x46904e
runtime.gcBgMarkWorker()
	/usr/local/go/src/runtime/mgc.go:1310 +0xe5 fp=0xc000194fe0 sp=0xc000194f50 pc=0x447045
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc000194fe8 sp=0xc000194fe0 pc=0x4a39e1
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/local/go/src/runtime/mgc.go:1234 +0x1c

goroutine 27 gp=0xc00018b880 m=nil [IO wait, 641 minutes]:
runtime.gopark(0x0?, 0x42b3e6?, 0x9b?, 0x73?, 0x521a040?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc0024039a0 sp=0xc002403980 pc=0x46904e
runtime.netpollblock(0x4a6cf1?, 0x576547?, 0x0?)
	/usr/local/go/src/runtime/netpoll.go:573 +0xf7 fp=0xc0024039d8 sp=0xc0024039a0 pc=0x460f57
internal/poll.runtime_pollWait(0x7158a0cc8d48, 0x72)
	/usr/local/go/src/runtime/netpoll.go:345 +0x85 fp=0xc0024039f8 sp=0xc0024039d8 pc=0x49d7e5
internal/poll.(*pollDesc).wait(0xc0001fe320, 0x72, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc002403a38 sp=0xc0024039f8 pc=0x56e351
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc0001fe300)
	/usr/local/go/src/internal/poll/fd_unix.go:611 +0x507 fp=0xc002403b40 sp=0xc002403a38 pc=0x576547
net.(*netFD).accept(0xc0001fe300)
	/usr/local/go/src/net/fd_unix.go:172 +0x45 fp=0xc002403c50 sp=0xc002403b40 pc=0x710d65
net.(*TCPListener).accept(0xc0008300c0)
	/usr/local/go/src/net/tcpsock_posix.go:159 +0x3e fp=0xc002403c98 sp=0xc002403c50 pc=0x73ff7e
net.(*TCPListener).Accept(0xc0008300c0)
	/usr/local/go/src/net/tcpsock.go:327 +0x65 fp=0xc002403d08 sp=0xc002403c98 pc=0x73de05
net/http.(*onceCloseListener).Accept(0xc00122e480)
	<autogenerated>:1 +0x44 fp=0xc002403d40 sp=0xc002403d08 pc=0x976484
net/http.(*Server).Serve(0xc000b4c1e0, {0x5230e00, 0xc0008300c0})
	/usr/local/go/src/net/http/server.go:3260 +0x672 fp=0xc002403f08 sp=0xc002403d40 pc=0x9375d2
net/http.(*Server).ListenAndServe(0xc000b4c1e0)
	/usr/local/go/src/net/http/server.go:3189 +0xbd fp=0xc002403f58 sp=0xc002403f08 pc=0x936edd
github.com/erigontech/erigon/turbo/debug.StartPProf.func1()
	/home/ubuntu/erigon/turbo/debug/flags.go:260 +0x32 fp=0xc002403fe0 sp=0xc002403f58 pc=0x3876ad2
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc002403fe8 sp=0xc002403fe0 pc=0x4a39e1
created by github.com/erigontech/erigon/turbo/debug.StartPProf in goroutine 1
	/home/ubuntu/erigon/turbo/debug/flags.go:259 +0x528

goroutine 50 gp=0xc0005a3340 m=nil [chan receive, 641 minutes]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x7c1e460?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc0000a0f08 sp=0xc0000a0ee8 pc=0x46904e
runtime.chanrecv(0xc0005da0e0, 0xc002405fbf, 0x1)
	/usr/local/go/src/runtime/chan.go:583 +0x36d fp=0xc0000a0f80 sp=0xc0000a0f08 pc=0x42e42d
runtime.chanrecv2(0xc001bfc540?, 0xc000bdc660?)
	/usr/local/go/src/runtime/chan.go:447 +0x12 fp=0xc0000a0fa8 sp=0xc0000a0f80 pc=0x42e0b2
gopkg.in/natefinch/lumberjack%2ev2.(*Logger).millRun(...)
	/home/ubuntu/go/pkg/mod/gopkg.in/natefinch/lumberjack.v2@v2.2.1/lumberjack.go:379
gopkg.in/natefinch/lumberjack%2ev2.(*Logger).mill.func1.gowrap1()
	/home/ubuntu/go/pkg/mod/gopkg.in/natefinch/lumberjack.v2@v2.2.1/lumberjack.go:390 +0x6b fp=0xc0000a0fe0 sp=0xc0000a0fa8 pc=0x29cae8b
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc0000a0fe8 sp=0xc0000a0fe0 pc=0x4a39e1
created by gopkg.in/natefinch/lumberjack%2ev2.(*Logger).mill.func1 in goroutine 1
	/home/ubuntu/go/pkg/mod/gopkg.in/natefinch/lumberjack.v2@v2.2.1/lumberjack.go:390 +0xea

goroutine 26 gp=0xc0005a3500 m=nil [IO wait, 250 minutes]:
runtime.gopark(0x0?, 0x42b3e6?, 0x9b?, 0x73?, 0x521a040?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc0024019b0 sp=0xc002401990 pc=0x46904e
runtime.netpollblock(0x4a6cf1?, 0x576547?, 0x0?)
	/usr/local/go/src/runtime/netpoll.go:573 +0xf7 fp=0xc0024019e8 sp=0xc0024019b0 pc=0x460f57
internal/poll.runtime_pollWait(0x7158a0cc8e40, 0x72)
	/usr/local/go/src/runtime/netpoll.go:345 +0x85 fp=0xc002401a08 sp=0xc0024019e8 pc=0x49d7e5
internal/poll.(*pollDesc).wait(0xc0001fe2a0, 0x72, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc002401a48 sp=0xc002401a08 pc=0x56e351
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc0001fe280)
	/usr/local/go/src/internal/poll/fd_unix.go:611 +0x507 fp=0xc002401b50 sp=0xc002401a48 pc=0x576547
net.(*netFD).accept(0xc0001fe280)
	/usr/local/go/src/net/fd_unix.go:172 +0x45 fp=0xc002401c60 sp=0xc002401b50 pc=0x710d65
net.(*TCPListener).accept(0xc000830080)
	/usr/local/go/src/net/tcpsock_posix.go:159 +0x3e fp=0xc002401ca8 sp=0xc002401c60 pc=0x73ff7e
net.(*TCPListener).Accept(0xc000830080)
	/usr/local/go/src/net/tcpsock.go:327 +0x65 fp=0xc002401d18 sp=0xc002401ca8 pc=0x73de05
net/http.(*onceCloseListener).Accept(0xc000db6750)
	<autogenerated>:1 +0x44 fp=0xc002401d50 sp=0xc002401d18 pc=0x976484
net/http.(*Server).Serve(0xc000b4c0f0, {0x5230e00, 0xc000830080})
	/usr/local/go/src/net/http/server.go:3260 +0x672 fp=0xc002401f18 sp=0xc002401d50 pc=0x9375d2
net/http.(*Server).ListenAndServe(0xc000b4c0f0)
	/usr/local/go/src/net/http/server.go:3189 +0xbd fp=0xc002401f68 sp=0xc002401f18 pc=0x936edd
github.com/erigontech/erigon-lib/metrics.Setup.func1()
	/home/ubuntu/erigon/erigon-lib/metrics/setup.go:45 +0x45 fp=0xc002401fe0 sp=0xc002401f68 pc=0xe2d385
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc002401fe8 sp=0xc002401fe0 pc=0x4a39e1
created by github.com/erigontech/erigon-lib/metrics.Setup in goroutine 1
	/home/ubuntu/erigon/erigon-lib/metrics/setup.go:44 +0x317

goroutine 66 gp=0xc0005a36c0 m=nil [IO wait]:
runtime.gopark(0x0?, 0x2ff?, 0x0?, 0x60?, 0x1000?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc023937638 sp=0xc023937618 pc=0x46904e
runtime.netpollblock(0x4a6cf1?, 0x5701a6?, 0x0?)
	/usr/local/go/src/runtime/netpoll.go:573 +0xf7 fp=0xc023937670 sp=0xc023937638 pc=0x460f57
internal/poll.runtime_pollWait(0x7158a0cc8c50, 0x72)
	/usr/local/go/src/runtime/netpoll.go:345 +0x85 fp=0xc023937690 sp=0xc023937670 pc=0x49d7e5
internal/poll.(*pollDesc).wait(0xc018e6d420, 0x72, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0239376d0 sp=0xc023937690 pc=0x56e351
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc018e6d400, {0xc023806000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:164 +0x466 fp=0xc0239377b8 sp=0xc0239376d0 pc=0x5701a6
net.(*netFD).Read(0xc018e6d400, {0xc023806000, 0x1000, 0x1000})
	/usr/local/go/src/net/fd_posix.go:55 +0x4b fp=0xc023937808 sp=0xc0239377b8 pc=0x70d78b
net.(*conn).Read(0xc01292a068, {0xc023806000, 0x1000, 0x1000})
	/usr/local/go/src/net/net.go:185 +0xad fp=0xc023937888 sp=0xc023937808 pc=0x72d4ed
net.(*TCPConn).Read(0xc01292a068, {0xc023806000, 0x1000, 0x1000})
	<autogenerated>:1 +0x54 fp=0xc0239378d0 sp=0xc023937888 pc=0x751a94
net/http.(*connReader).Read(0xc023802030, {0xc023806000, 0x1000, 0x1000})
	/usr/local/go/src/net/http/server.go:789 +0x212 fp=0xc023937958 sp=0xc0239378d0 pc=0x9252b2
bufio.(*Reader).fill(0xc01bf08180)
	/usr/local/go/src/bufio/bufio.go:110 +0x2b0 fp=0xc0239379f0 sp=0xc023937958 pc=0x84d5f0
bufio.(*Reader).Peek(0xc01bf08180, 0x4)
	/usr/local/go/src/bufio/bufio.go:148 +0xc7 fp=0xc023937a78 sp=0xc0239379f0 pc=0x84d8a7
net/http.(*conn).serve(0xc000d4a120, {0x5238ff0, 0xc000db6e70})
	/usr/local/go/src/net/http/server.go:2079 +0x161c fp=0xc023937fa0 sp=0xc023937a78 pc=0x92ff5c
net/http.(*Server).Serve.gowrap3()
	/usr/local/go/src/net/http/server.go:3290 +0x50 fp=0xc023937fe0 sp=0xc023937fa0 pc=0x937bb0
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc023937fe8 sp=0xc023937fe0 pc=0x4a39e1
created by net/http.(*Server).Serve in goroutine 26
	/usr/local/go/src/net/http/server.go:3290 +0x8ed

goroutine 53000638 gp=0xc000602700 m=nil [select]:
runtime.gopark(0xc168233f08?, 0x2?, 0x20?, 0x3d?, 0xc168233e6c?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc168233cc8 sp=0xc168233ca8 pc=0x46904e
runtime.selectgo(0xc168233f08, 0xc168233e68, 0xc0c7c39eb8?, 0x0, 0xc0c7c39e50?, 0x1)
	/usr/local/go/src/runtime/select.go:327 +0x865 fp=0xc168233e18 sp=0xc168233cc8 pc=0x47c225
github.com/libp2p/go-libp2p-pubsub.(*PubSub).handleSendingMessages(0x1?, {0x5238b80, 0x8bc5de0}, {0x5259a50, 0xc1332db2c0}, 0xc13933b560)
	/home/ubuntu/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.11.0/comm.go:178 +0x1f0 fp=0xc168233f70 sp=0xc168233e18 pc=0x3416230
github.com/libp2p/go-libp2p-pubsub.(*PubSub).handleNewPeer.gowrap1()
	/home/ubuntu/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.11.0/comm.go:130 +0x7a fp=0xc168233fe0 sp=0xc168233f70 pc=0x3415cba
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc168233fe8 sp=0xc168233fe0 pc=0x4a39e1
created by github.com/libp2p/go-libp2p-pubsub.(*PubSub).handleNewPeer in goroutine 53000636
	/home/ubuntu/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.11.0/comm.go:130 +0x51d

goroutine 194 gp=0xc0006028c0 m=nil [chan receive, 641 minutes]:
runtime.gopark(0xc0396f3ee0?, 0x5af1c5?, 0x8?, 0x6?, 0x1c?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc0396f3e50 sp=0xc0396f3e30 pc=0x46904e
runtime.chanrecv(0xc000110360, 0x0, 0x1)
	/usr/local/go/src/runtime/chan.go:583 +0x36d fp=0xc0396f3ec8 sp=0xc0396f3e50 pc=0x42e42d
runtime.chanrecv1(0xc0012905f0?, 0x5239028?)
	/usr/local/go/src/runtime/chan.go:442 +0x12 fp=0xc0396f3ef0 sp=0xc0396f3ec8 pc=0x42e092
github.com/erigontech/erigon/cmd/rpcdaemon/cli.stopAuthenticatedRpcServer({0x5239028, 0xc0012905f0}, 0xc0324e1380, {0x524ec28, 0xc000b6d120})
	/home/ubuntu/erigon/cmd/rpcdaemon/cli/config.go:876 +0xe6 fp=0xc0396f3f80 sp=0xc0396f3ef0 pc=0x3da5bc6
github.com/erigontech/erigon/cmd/rpcdaemon/cli.StartRpcServerWithJwtAuthentication.gowrap1()
	/home/ubuntu/erigon/cmd/rpcdaemon/cli/config.go:664 +0x6c fp=0xc0396f3fe0 sp=0xc0396f3f80 pc=0x3da140c
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc0396f3fe8 sp=0xc0396f3fe0 pc=0x4a39e1
created by github.com/erigontech/erigon/cmd/rpcdaemon/cli.StartRpcServerWithJwtAuthentication in goroutine 106
	/home/ubuntu/erigon/cmd/rpcdaemon/cli/config.go:664 +0x196

goroutine 33 gp=0xc000602a80 m=nil [IO wait, 641 minutes]:
runtime.gopark(0x0?, 0x42b3e6?, 0x9b?, 0x73?, 0x521a040?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc0000a19b0 sp=0xc0000a1990 pc=0x46904e
runtime.netpollblock(0x4a6cf1?, 0x576547?, 0x0?)
	/usr/local/go/src/runtime/netpoll.go:573 +0xf7 fp=0xc0000a19e8 sp=0xc0000a19b0 pc=0x460f57
internal/poll.runtime_pollWait(0x7158a0cc8a60, 0x72)
	/usr/local/go/src/runtime/netpoll.go:345 +0x85 fp=0xc0000a1a08 sp=0xc0000a19e8 pc=0x49d7e5
internal/poll.(*pollDesc).wait(0xc0324e23a0, 0x72, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0000a1a48 sp=0xc0000a1a08 pc=0x56e351
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc0324e2380)
	/usr/local/go/src/internal/poll/fd_unix.go:611 +0x507 fp=0xc0000a1b50 sp=0xc0000a1a48 pc=0x576547
net.(*netFD).accept(0xc0324e2380)
	/usr/local/go/src/net/fd_unix.go:172 +0x45 fp=0xc0000a1c60 sp=0xc0000a1b50 pc=0x710d65
net.(*TCPListener).accept(0xc0396f5b00)
	/usr/local/go/src/net/tcpsock_posix.go:159 +0x3e fp=0xc0000a1ca8 sp=0xc0000a1c60 pc=0x73ff7e
net.(*TCPListener).Accept(0xc0396f5b00)
	/usr/local/go/src/net/tcpsock.go:327 +0x65 fp=0xc0000a1d18 sp=0xc0000a1ca8 pc=0x73de05
net/http.(*onceCloseListener).Accept(0xc031d9a0c0)
	<autogenerated>:1 +0x44 fp=0xc0000a1d50 sp=0xc0000a1d18 pc=0x976484
net/http.(*Server).Serve(0xc0396f80f0, {0x5230e00, 0xc0396f5b00})
	/usr/local/go/src/net/http/server.go:3260 +0x672 fp=0xc0000a1f18 sp=0xc0000a1d50 pc=0x9375d2
github.com/erigontech/erigon/node.StartHTTPEndpoint.func1()
	/home/ubuntu/erigon/node/endpoints.go:84 +0x1d2 fp=0xc0000a1fe0 sp=0xc0000a1f18 pc=0x3a246b2
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc0000a1fe8 sp=0xc0000a1fe0 pc=0x4a39e1
created by github.com/erigontech/erigon/node.StartHTTPEndpoint in goroutine 106
	/home/ubuntu/erigon/node/endpoints.go:76 +0x645

goroutine 67 gp=0xc000602c40 m=nil [chan receive, 641 minutes]:
runtime.gopark(0xc023883ea8?, 0x5af1c5?, 0x8?, 0x6?, 0x0?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc023883e18 sp=0xc023883df8 pc=0x46904e
runtime.chanrecv(0xc000110360, 0x0, 0x1)
	/usr/local/go/src/runtime/chan.go:583 +0x36d fp=0xc023883e90 sp=0xc023883e18 pc=0x42e42d
runtime.chanrecv1(0xc0012905f0?, 0x5245a60?)
	/usr/local/go/src/runtime/chan.go:442 +0x12 fp=0xc023883eb8 sp=0xc023883e90 pc=0x42e092
github.com/erigontech/erigon/ethdb/privateapi.(*MiningServer).OnPendingLogs(0xc000d4a3f0, 0x0?, {0x5245a60, 0xc0320420c0})
	/home/ubuntu/erigon/ethdb/privateapi/mining.go:112 +0x94 fp=0xc023883f30 sp=0xc023883eb8 pc=0x3826eb4
github.com/erigontech/erigon-lib/direct.(*MiningClient).OnPendingLogs.func1()
	/home/ubuntu/erigon/erigon-lib/direct/mining_client.go:152 +0xe3 fp=0xc023883fe0 sp=0xc023883f30 pc=0x14138a3
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc023883fe8 sp=0xc023883fe0 pc=0x4a39e1
created by github.com/erigontech/erigon-lib/direct.(*MiningClient).OnPendingLogs in goroutine 104
	/home/ubuntu/erigon/erigon-lib/direct/mining_client.go:150 +0x23d

goroutine 42 gp=0xc023888000 m=nil [select, 641 minutes]:
runtime.gopark(0xc0000a5f00?, 0x2?, 0x0?, 0x80?, 0xc0000a5e8c?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc0000a5d08 sp=0xc0000a5ce8 pc=0x46904e
runtime.selectgo(0xc0000a5f00, 0xc0000a5e88, 0xc031878090?, 0x0, 0x4a6d29?, 0x1)
	/usr/local/go/src/runtime/select.go:327 +0x865 fp=0xc0000a5e58 sp=0xc0000a5d08 pc=0x47c225
github.com/erigontech/erigon/p2p/sentry.(*GrpcServer).PeerEvents(0xc0001c4000, 0x0?, {0x5245920, 0xc031878090})
	/home/ubuntu/erigon/p2p/sentry/sentry_grpc_server.go:1256 +0x165 fp=0xc0000a5f30 sp=0xc0000a5e58 pc=0x3900205
github.com/erigontech/erigon-lib/direct.(*SentryClientDirect).PeerEvents.func1()
	/home/ubuntu/erigon/erigon-lib/direct/sentry_client.go:255 +0xe3 fp=0xc0000a5fe0 sp=0xc0000a5f30 pc=0x1418283
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc0000a5fe8 sp=0xc0000a5fe0 pc=0x4a39e1
created by github.com/erigontech/erigon-lib/direct.(*SentryClientDirect).PeerEvents in goroutine 92
	/home/ubuntu/erigon/erigon-lib/direct/sentry_client.go:253 +0x23d

goroutine 195 gp=0xc000602e00 m=nil [select]:
runtime.gopark(0xc0318ecf40?, 0x3?, 0x0?, 0x2e?, 0xc0318ecea2?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc0318ecd08 sp=0xc0318ecce8 pc=0x46904e
runtime.selectgo(0xc0318ecf40, 0xc0318ece9c, 0xc0324e1b00?, 0x0, 0x3?, 0x1)
	/usr/local/go/src/runtime/select.go:327 +0x865 fp=0xc0318ece58 sp=0xc0318ecd08 pc=0x47c225
github.com/erigontech/erigon/p2p/discover.(*UDPv4).loop.func1()
	/home/ubuntu/erigon/p2p/discover/v4_udp.go:548 +0x326 fp=0xc0318ecfe0 sp=0xc0318ece58 pc=0x29518e6
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc0318ecfe8 sp=0xc0318ecfe0 pc=0x4a39e1
created by github.com/erigontech/erigon/p2p/discover.(*UDPv4).loop in goroutine 289
	/home/ubuntu/erigon/p2p/discover/v4_udp.go:511 +0x477

goroutine 84 gp=0xc000602fc0 m=nil [select, 2 minutes]:
runtime.gopark(0xc031d95f08?, 0x2?, 0xc0?, 0x2f?, 0xc031d95d78?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc031d95bf0 sp=0xc031d95bd0 pc=0x46904e
runtime.selectgo(0xc031d95f08, 0xc031d95d74, 0xf?, 0x0, 0x4?, 0x1)
	/usr/local/go/src/runtime/select.go:327 +0x865 fp=0xc031d95d40 sp=0xc031d95bf0 pc=0x47c225
github.com/erigontech/erigon/eth.New.func6()
	/home/ubuntu/erigon/eth/backend.go:477 +0x1ba fp=0xc031d95fe0 sp=0xc031d95d40 pc=0x3e320ba
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc031d95fe8 sp=0xc031d95fe0 pc=0x4a39e1
created by github.com/erigontech/erigon/eth.New in goroutine 1
	/home/ubuntu/erigon/eth/backend.go:470 +0xe18b

goroutine 236 gp=0xc000603180 m=nil [select]:
runtime.gopark(0xc189b17f90?, 0x2?, 0x80?, 0x31?, 0xc189b17f14?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc189b17d90 sp=0xc189b17d70 pc=0x46904e
runtime.selectgo(0xc189b17f90, 0xc189b17f10, 0xc16706b2c0?, 0x0, 0xc00099e330?, 0x1)
	/usr/local/go/src/runtime/select.go:327 +0x865 fp=0xc189b17ee0 sp=0xc189b17d90 pc=0x47c225
github.com/erigontech/erigon-lib/p2p/sentry.pumpStreamLoop[...].func2()
	/home/ubuntu/erigon/erigon-lib/p2p/sentry/loop.go:122 +0x12a fp=0xc189b17fe0 sp=0xc189b17ee0 pc=0x3dbfe0a
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc189b17fe8 sp=0xc189b17fe0 pc=0x4a39e1
created by github.com/erigontech/erigon-lib/p2p/sentry.pumpStreamLoop[...] in goroutine 306
	/home/ubuntu/erigon/erigon-lib/p2p/sentry/loop.go:120 +0x557

goroutine 147 gp=0xc000603340 m=nil [IO wait, 641 minutes]:
runtime.gopark(0x0?, 0x42b3e6?, 0x9b?, 0x73?, 0x521a040?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc00009f9b0 sp=0xc00009f990 pc=0x46904e
runtime.netpollblock(0x4a6cf1?, 0x576547?, 0x0?)
	/usr/local/go/src/runtime/netpoll.go:573 +0xf7 fp=0xc00009f9e8 sp=0xc00009f9b0 pc=0x460f57
internal/poll.runtime_pollWait(0x7158a0cc8870, 0x72)
	/usr/local/go/src/runtime/netpoll.go:345 +0x85 fp=0xc00009fa08 sp=0xc00009f9e8 pc=0x49d7e5
internal/poll.(*pollDesc).wait(0xc02392a920, 0x72, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00009fa48 sp=0xc00009fa08 pc=0x56e351
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc02392a900)
	/usr/local/go/src/internal/poll/fd_unix.go:611 +0x507 fp=0xc00009fb50 sp=0xc00009fa48 pc=0x576547
net.(*netFD).accept(0xc02392a900)
	/usr/local/go/src/net/fd_unix.go:172 +0x45 fp=0xc00009fc60 sp=0xc00009fb50 pc=0x710d65
net.(*TCPListener).accept(0xc0170cb6e0)
	/usr/local/go/src/net/tcpsock_posix.go:159 +0x3e fp=0xc00009fca8 sp=0xc00009fc60 pc=0x73ff7e
net.(*TCPListener).Accept(0xc0170cb6e0)
	/usr/local/go/src/net/tcpsock.go:327 +0x65 fp=0xc00009fd18 sp=0xc00009fca8 pc=0x73de05
net/http.(*onceCloseListener).Accept(0xc031d9a1b0)
	<autogenerated>:1 +0x44 fp=0xc00009fd50 sp=0xc00009fd18 pc=0x976484
net/http.(*Server).Serve(0xc0170e0000, {0x5230e00, 0xc0170cb6e0})
	/usr/local/go/src/net/http/server.go:3260 +0x672 fp=0xc00009ff18 sp=0xc00009fd50 pc=0x9375d2
github.com/erigontech/erigon/node.StartHTTPEndpoint.func1()
	/home/ubuntu/erigon/node/endpoints.go:84 +0x1d2 fp=0xc00009ffe0 sp=0xc00009ff18 pc=0x3a246b2
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc00009ffe8 sp=0xc00009ffe0 pc=0x4a39e1
created by github.com/erigontech/erigon/node.StartHTTPEndpoint in goroutine 105
	/home/ubuntu/erigon/node/endpoints.go:76 +0x645

goroutine 114 gp=0xc000603500 m=nil [select]:
runtime.gopark(0xc0396ecee8?, 0x3?, 0x0?, 0x35?, 0xc0396ece62?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc0396eccd8 sp=0xc0396eccb8 pc=0x46904e
runtime.selectgo(0xc0396ecee8, 0xc0396ece5c, 0x0?, 0x0, 0x0?, 0x1)
	/usr/local/go/src/runtime/select.go:327 +0x865 fp=0xc0396ece28 sp=0xc0396eccd8 pc=0x47c225
github.com/erigontech/erigon-lib/kv/remotedbserver.(*KvServer).StateChanges(0xc000c20090, 0x0?, {0x5245880, 0xc0318f4030})
	/home/ubuntu/erigon/erigon-lib/kv/remotedbserver/remotedbserver.go:437 +0x1d1 fp=0xc0396ecf30 sp=0xc0396ece28 pc=0x2a58051
github.com/erigontech/erigon-lib/direct.(*StateDiffClientDirect).StateChanges.func1()
	/home/ubuntu/erigon/erigon-lib/direct/state_diff_client.go:55 +0xe3 fp=0xc0396ecfe0 sp=0xc0396ecf30 pc=0x14190c3
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc0396ecfe8 sp=0xc0396ecfe0 pc=0x4a39e1
created by github.com/erigontech/erigon-lib/direct.(*StateDiffClientDirect).StateChanges in goroutine 90
	/home/ubuntu/erigon/erigon-lib/direct/state_diff_client.go:53 +0x23d

goroutine 8008 gp=0xc0006036c0 m=nil [IO wait]:
runtime.gopark(0x0?, 0x4fd?, 0x0?, 0x60?, 0x4000?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc03ed29708 sp=0xc03ed296e8 pc=0x46904e
runtime.netpollblock(0x4a6cf1?, 0x5701a6?, 0x0?)
	/usr/local/go/src/runtime/netpoll.go:573 +0xf7 fp=0xc03ed29740 sp=0xc03ed29708 pc=0x460f57
internal/poll.runtime_pollWait(0x7158a01fa6b8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:345 +0x85 fp=0xc03ed29760 sp=0xc03ed29740 pc=0x49d7e5
internal/poll.(*pollDesc).wait(0xc0c833e220, 0x72, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc03ed297a0 sp=0xc03ed29760 pc=0x56e351
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0c833e200, {0xc100c06000, 0x4000, 0x4000})
	/usr/local/go/src/internal/poll/fd_unix.go:164 +0x466 fp=0xc03ed29888 sp=0xc03ed297a0 pc=0x5701a6
net.(*netFD).Read(0xc0c833e200, {0xc100c06000, 0x4000, 0x4000})
	/usr/local/go/src/net/fd_posix.go:55 +0x4b fp=0xc03ed298d8 sp=0xc03ed29888 pc=0x70d78b
net.(*conn).Read(0xc0375024a8, {0xc100c06000, 0x4000, 0x4000})
	/usr/local/go/src/net/net.go:185 +0xad fp=0xc03ed29958 sp=0xc03ed298d8 pc=0x72d4ed
net.(*TCPConn).Read(0xc0375024a8, {0xc100c06000, 0x4000, 0x4000})
	<autogenerated>:1 +0x54 fp=0xc03ed299a0 sp=0xc03ed29958 pc=0x751a94
github.com/erigontech/erigon/p2p.(*meteredConn).Read(0xc0e70832b0, {0xc100c06000, 0x4000, 0x4000})
	/home/ubuntu/erigon/p2p/metrics.go:66 +0x62 fp=0xc03ed299e8 sp=0xc03ed299a0 pc=0x2984482
io.ReadAtLeast({0x7158a07d0818, 0xc0e70832b0}, {0xc100c06000, 0x4000, 0x4000}, 0x20)
	/usr/local/go/src/io/io.go:335 +0xd0 fp=0xc03ed29a30 sp=0xc03ed299e8 pc=0x52acf0
github.com/erigontech/erigon/p2p/rlpx.(*readBuffer).read(0xc09c776b00, {0x7158a07d0818, 0xc0e70832b0}, 0x20)
	/home/ubuntu/erigon/p2p/rlpx/buffer.go:67 +0x44f fp=0xc03ed29ad8 sp=0xc03ed29a30 pc=0x297402f
github.com/erigontech/erigon/p2p/rlpx.(*sessionState).readFrame(0xc09c776a00, {0x7158a07d0818, 0xc0e70832b0})
	/home/ubuntu/erigon/p2p/rlpx/rlpx.go:170 +0x23d fp=0xc03ed29c10 sp=0xc03ed29ad8 pc=0x297513d
github.com/erigontech/erigon/p2p/rlpx.(*Conn).Read(0xc0ce7008c0)
	/home/ubuntu/erigon/p2p/rlpx/rlpx.go:140 +0xdb fp=0xc03ed29cf0 sp=0xc03ed29c10 pc=0x2974afb
github.com/erigontech/erigon/p2p.(*rlpxTransport).ReadMsg(0xc0ce70a440)
	/home/ubuntu/erigon/p2p/transport.go:71 +0x1a9 fp=0xc03ed29e50 sp=0xc03ed29cf0 pc=0x299bee9
github.com/erigontech/erigon/p2p.(*Peer).readLoop(0xc039809290, 0xc0c9967200)
	/home/ubuntu/erigon/p2p/peer.go:339 +0x162 fp=0xc03ed29fb0 sp=0xc03ed29e50 pc=0x29873e2
github.com/erigontech/erigon/p2p.(*Peer).run.gowrap1()
	/home/ubuntu/erigon/p2p/peer.go:275 +0x45 fp=0xc03ed29fe0 sp=0xc03ed29fb0 pc=0x2986ba5
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc03ed29fe8 sp=0xc03ed29fe0 pc=0x4a39e1
created by github.com/erigontech/erigon/p2p.(*Peer).run in goroutine 8033
	/home/ubuntu/erigon/p2p/peer.go:275 +0x176

goroutine 52 gp=0xc000603880 m=nil [select]:
runtime.gopark(0xc032881eb0?, 0x4?, 0x80?, 0x38?, 0xc032881d70?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc032881be8 sp=0xc032881bc8 pc=0x46904e
runtime.selectgo(0xc032881eb0, 0xc032881d68, 0x2a?, 0x0, 0x0?, 0x1)
	/usr/local/go/src/runtime/select.go:327 +0x865 fp=0xc032881d38 sp=0xc032881be8 pc=0x47c225
github.com/erigontech/erigon/ethdb/privateapi.(*EthBackendServer).Subscribe(0xc0005fea10, 0xd7616aa?, {0x52456a0, 0xc0320d6090})
	/home/ubuntu/erigon/ethdb/privateapi/ethbackend.go:242 +0x4bb fp=0xc032881f30 sp=0xc032881d38 pc=0x381a3fb
github.com/erigontech/erigon-lib/direct.(*EthBackendClientDirect).Subscribe.func1()
	/home/ubuntu/erigon/erigon-lib/direct/eth_backend_client.go:73 +0xe6 fp=0xc032881fe0 sp=0xc032881f30 pc=0x140f366
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc032881fe8 sp=0xc032881fe0 pc=0x4a39e1
created by github.com/erigontech/erigon-lib/direct.(*EthBackendClientDirect).Subscribe in goroutine 100
	/home/ubuntu/erigon/erigon-lib/direct/eth_backend_client.go:71 +0x23d

goroutine 162 gp=0xc000603c00 m=nil [select]:
runtime.gopark(0xc031be7ed8?, 0x3?, 0x0?, 0x3c?, 0xc031be7dd2?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc031be7c48 sp=0xc031be7c28 pc=0x46904e
runtime.selectgo(0xc031be7ed8, 0xc031be7dcc, 0x4?, 0x0, 0xc013587500?, 0x1)
	/usr/local/go/src/runtime/select.go:327 +0x865 fp=0xc031be7d98 sp=0xc031be7c48 pc=0x47c225
github.com/erigontech/erigon/p2p/sentry.(*GrpcServer).Messages(0xc0001c40e0, 0xc037cf6080, {0x52459c0, 0xc031ca8060})
	/home/ubuntu/erigon/p2p/sentry/sentry_grpc_server.go:1219 +0x3ec fp=0xc031be7f30 sp=0xc031be7d98 pc=0x38ff7ec
github.com/erigontech/erigon-lib/direct.(*SentryClientDirect).Messages.func1()
	/home/ubuntu/erigon/erigon-lib/direct/sentry_client.go:190 +0xe3 fp=0xc031be7fe0 sp=0xc031be7f30 pc=0x14176e3
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc031be7fe8 sp=0xc031be7fe0 pc=0x4a39e1
created by github.com/erigontech/erigon-lib/direct.(*SentryClientDirect).Messages in goroutine 93
	/home/ubuntu/erigon/erigon-lib/direct/sentry_client.go:188 +0x4cd

goroutine 85 gp=0xc000603dc0 m=nil [select, 2 minutes]:
runtime.gopark(0xc023933ef0?, 0x2?, 0xc0?, 0x3d?, 0xc023932674?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc0239324c0 sp=0xc0239324a0 pc=0x46904e
runtime.selectgo(0xc023933ef0, 0xc023932670, 0x87549a000?, 0x0, 0x872a47800?, 0x1)
	/usr/local/go/src/runtime/select.go:327 +0x865 fp=0xc023932610 sp=0xc0239324c0 pc=0x47c225
github.com/erigontech/erigon-lib/common/mem.LogMemStats({0x5239028, 0xc0012905f0}, {0x524ec28, 0xc000b6d120})
	/home/ubuntu/erigon/erigon-lib/common/mem/common.go:68 +0x1ac fp=0xc023933f90 sp=0xc023932610 pc=0x29e684c
github.com/erigontech/erigon/eth.New.gowrap1()
	/home/ubuntu/erigon/eth/backend.go:503 +0x5e fp=0xc023933fe0 sp=0xc023933f90 pc=0x3e31ebe
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc023933fe8 sp=0xc023933fe0 pc=0x4a39e1
created by github.com/erigontech/erigon/eth.New in goroutine 1
	/home/ubuntu/erigon/eth/backend.go:503 +0x3167

goroutine 86 gp=0xc0238881c0 m=nil [select]:
runtime.gopark(0xc093ecbf50?, 0x2?, 0xc0?, 0x81?, 0xc093ecbed4?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc093ecbd50 sp=0xc093ecbd30 pc=0x46904e
runtime.selectgo(0xc093ecbf50, 0xc093ecbed0, 0x0?, 0x0, 0x0?, 0x1)
	/usr/local/go/src/runtime/select.go:327 +0x865 fp=0xc093ecbea0 sp=0xc093ecbd50 pc=0x47c225
github.com/erigontech/erigon-lib/common/disk.UpdateDiskStats({0x5239028, 0xc0012905f0}, {0x524ec28, 0xc000b6d120})
	/home/ubuntu/erigon/erigon-lib/common/disk/common.go:31 +0x185 fp=0xc093ecbf90 sp=0xc093ecbea0 pc=0x29e5945
github.com/erigontech/erigon/eth.New.gowrap2()
	/home/ubuntu/erigon/eth/backend.go:504 +0x5e fp=0xc093ecbfe0 sp=0xc093ecbf90 pc=0x3e31e1e
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc093ecbfe8 sp=0xc093ecbfe0 pc=0x4a39e1
created by github.com/erigontech/erigon/eth.New in goroutine 1
	/home/ubuntu/erigon/eth/backend.go:504 +0x3265

goroutine 146 gp=0xc023888380 m=nil [select, 641 minutes]:
runtime.gopark(0xc023882f00?, 0x2?, 0x80?, 0x83?, 0xc023882e8c?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc023882d08 sp=0xc023882ce8 pc=0x46904e
runtime.selectgo(0xc023882f00, 0xc023882e88, 0xc00f3978f0?, 0x0, 0x0?, 0x1)
	/usr/local/go/src/runtime/select.go:327 +0x865 fp=0xc023882e58 sp=0xc023882d08 pc=0x47c225
github.com/erigontech/erigon/p2p/sentry.(*GrpcServer).PeerEvents(0xc0001c40e0, 0xa92fd2?, {0x5245920, 0xc00f3978f0})
	/home/ubuntu/erigon/p2p/sentry/sentry_grpc_server.go:1256 +0x165 fp=0xc023882f30 sp=0xc023882e58 pc=0x3900205
github.com/erigontech/erigon-lib/direct.(*SentryClientDirect).PeerEvents.func1()
	/home/ubuntu/erigon/erigon-lib/direct/sentry_client.go:255 +0xe3 fp=0xc023882fe0 sp=0xc023882f30 pc=0x1418283
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc023882fe8 sp=0xc023882fe0 pc=0x4a39e1
created by github.com/erigontech/erigon-lib/direct.(*SentryClientDirect).PeerEvents in goroutine 94
	/home/ubuntu/erigon/erigon-lib/direct/sentry_client.go:253 +0x23d

goroutine 88 gp=0xc0238888c0 m=nil [select, 641 minutes]:
runtime.gopark(0xc002405fb0?, 0x2?, 0xc0?, 0x88?, 0xc002405eb4?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc002405d30 sp=0xc002405d10 pc=0x46904e
runtime.selectgo(0xc002405fb0, 0xc002405eb0, 0x24?, 0x0, 0x0?, 0x1)
	/usr/local/go/src/runtime/select.go:327 +0x865 fp=0xc002405e80 sp=0xc002405d30 pc=0x47c225
github.com/erigontech/erigon/ethdb/privateapi.NewEthBackendServer.func1()
	/home/ubuntu/erigon/ethdb/privateapi/ethbackend.go:105 +0x28c fp=0xc002405fe0 sp=0xc002405e80 pc=0x3817c4c
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc002405fe8 sp=0xc002405fe0 pc=0x4a39e1
created by github.com/erigontech/erigon/ethdb/privateapi.NewEthBackendServer in goroutine 1
	/home/ubuntu/erigon/ethdb/privateapi/ethbackend.go:91 +0x653

goroutine 89 gp=0xc023888c40 m=nil [IO wait, 641 minutes]:
runtime.gopark(0x0?, 0x42b3e6?, 0x9b?, 0x73?, 0x521a040?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc031d91988 sp=0xc031d91968 pc=0x46904e
runtime.netpollblock(0x4a6cf1?, 0x576547?, 0x0?)
	/usr/local/go/src/runtime/netpoll.go:573 +0xf7 fp=0xc031d919c0 sp=0xc031d91988 pc=0x460f57
internal/poll.runtime_pollWait(0x7158a0cc8b58, 0x72)
	/usr/local/go/src/runtime/netpoll.go:345 +0x85 fp=0xc031d919e0 sp=0xc031d919c0 pc=0x49d7e5
internal/poll.(*pollDesc).wait(0xc031586820, 0x72, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc031d91a20 sp=0xc031d919e0 pc=0x56e351
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc031586800)
	/usr/local/go/src/internal/poll/fd_unix.go:611 +0x507 fp=0xc031d91b28 sp=0xc031d91a20 pc=0x576547
net.(*netFD).accept(0xc031586800)
	/usr/local/go/src/net/fd_unix.go:172 +0x45 fp=0xc031d91c38 sp=0xc031d91b28 pc=0x710d65
net.(*TCPListener).accept(0xc0317a44a0)
	/usr/local/go/src/net/tcpsock_posix.go:159 +0x3e fp=0xc031d91c80 sp=0xc031d91c38 pc=0x73ff7e
net.(*TCPListener).Accept(0xc0317a44a0)
	/usr/local/go/src/net/tcpsock.go:327 +0x65 fp=0xc031d91cf0 sp=0xc031d91c80 pc=0x73de05
google.golang.org/grpc.(*Server).Serve(0xc03740d000, {0x5230e00, 0xc0317a44a0})
	/home/ubuntu/go/pkg/mod/google.golang.org/grpc@v1.65.0/server.go:885 +0x80b fp=0xc031d91f08 sp=0xc031d91cf0 pc=0x1351bab
github.com/erigontech/erigon/ethdb/privateapi.StartGrpc.func1()
	/home/ubuntu/erigon/ethdb/privateapi/all.go:72 +0xf5 fp=0xc031d91fe0 sp=0xc031d91f08 pc=0x38170f5
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc031d91fe8 sp=0xc031d91fe0 pc=0x4a39e1
created by github.com/erigontech/erigon/ethdb/privateapi.StartGrpc in goroutine 1
	/home/ubuntu/erigon/ethdb/privateapi/all.go:68 +0x6e5

goroutine 90 gp=0xc023888e00 m=nil [chan receive]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc02f425da0 sp=0xc02f425d80 pc=0x46904e
runtime.chanrecv(0xc01b998c60, 0xc02f425e60, 0x1)
	/usr/local/go/src/runtime/chan.go:583 +0x36d fp=0xc02f425e18 sp=0xc02f425da0 pc=0x42e42d
runtime.chanrecv2(0xf?, 0xc03433fc70?)
	/usr/local/go/src/runtime/chan.go:447 +0x12 fp=0xc02f425e40 sp=0xc02f425e18 pc=0x42e0b2
github.com/erigontech/erigon-lib/direct.(*StateDiffStreamC).Recv(0xc0318f4090)
	/home/ubuntu/erigon/erigon-lib/direct/state_diff_client.go:72 +0x4c fp=0xc02f425e88 sp=0xc02f425e40 pc=0x141928c
github.com/erigontech/erigon/txnprovider/txpool.(*Fetch).handleStateChanges(0xc0374ad5e0, {0x5239028, 0xc0012905f0}, {0x521bb20, 0xc037480eb0})
	/home/ubuntu/erigon/txnprovider/txpool/fetch.go:457 +0x270 fp=0xc02f425f70 sp=0xc02f425e88 pc=0x1e9c290
github.com/erigontech/erigon/txnprovider/txpool.(*Fetch).ConnectCore.func1()
	/home/ubuntu/erigon/txnprovider/txpool/fetch.go:120 +0xa5 fp=0xc02f425fe0 sp=0xc02f425f70 pc=0x1e97ea5
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc02f425fe8 sp=0xc02f425fe0 pc=0x4a39e1
created by github.com/erigontech/erigon/txnprovider/txpool.(*Fetch).ConnectCore in goroutine 1
	/home/ubuntu/erigon/txnprovider/txpool/fetch.go:113 +0x8e

goroutine 91 gp=0xc023888fc0 m=nil [chan receive]:
runtime.gopark(0xc12b068420?, 0xc031d83bd8?, 0x30?, 0xa9?, 0x1?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc03ab65ce0 sp=0xc03ab65cc0 pc=0x46904e
runtime.chanrecv(0xc0168441e0, 0xc03ab65da0, 0x1)
	/usr/local/go/src/runtime/chan.go:583 +0x36d fp=0xc03ab65d58 sp=0xc03ab65ce0 pc=0x42e42d
runtime.chanrecv2(0xc0319ca000?, 0xc00a94e010?)
	/usr/local/go/src/runtime/chan.go:447 +0x12 fp=0xc03ab65d80 sp=0xc03ab65d58 pc=0x42e0b2
github.com/erigontech/erigon-lib/direct.(*SentryMessagesStreamC).Recv(0xc0319c80c0)
	/home/ubuntu/erigon/erigon-lib/direct/sentry_client.go:228 +0x4c fp=0xc03ab65dc8 sp=0xc03ab65d80 pc=0x1417b8c
github.com/erigontech/erigon/txnprovider/txpool.(*Fetch).receiveMessage(0xc0374ad5e0, {0x5239028, 0xc0012905f0}, {0x5258a20, 0xc00099e318})
	/home/ubuntu/erigon/txnprovider/txpool/fetch.go:176 +0x3ff fp=0xc03ab65ee8 sp=0xc03ab65dc8 pc=0x1e9889f
github.com/erigontech/erigon/txnprovider/txpool.(*Fetch).receiveMessageLoop(0xc0374ad5e0, {0x5258a20, 0xc00099e318})
	/home/ubuntu/erigon/txnprovider/txpool/fetch.go:147 +0x2de fp=0xc03ab65f78 sp=0xc03ab65ee8 pc=0x1e982fe
github.com/erigontech/erigon/txnprovider/txpool.(*Fetch).ConnectSentries.func1(0x0)
	/home/ubuntu/erigon/txnprovider/txpool/fetch.go:104 +0x8b fp=0xc03ab65fb8 sp=0xc03ab65f78 pc=0x1e97d0b
github.com/erigontech/erigon/txnprovider/txpool.(*Fetch).ConnectSentries.gowrap1()
	/home/ubuntu/erigon/txnprovider/txpool/fetch.go:105 +0x42 fp=0xc03ab65fe0 sp=0xc03ab65fb8 pc=0x1e97c42
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc03ab65fe8 sp=0xc03ab65fe0 pc=0x4a39e1
created by github.com/erigontech/erigon/txnprovider/txpool.(*Fetch).ConnectSentries in goroutine 1
	/home/ubuntu/erigon/txnprovider/txpool/fetch.go:103 +0x17d

goroutine 92 gp=0xc023889180 m=nil [chan receive, 2 minutes]:
runtime.gopark(0xc0396263c0?, 0x0?, 0x29?, 0x6d?, 0xc0396efdc8?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc0396efd10 sp=0xc0396efcf0 pc=0x46904e
runtime.chanrecv(0xc039626360, 0xc0396efdd0, 0x1)
	/usr/local/go/src/runtime/chan.go:583 +0x36d fp=0xc0396efd88 sp=0xc0396efd10 pc=0x42e42d
runtime.chanrecv2(0xc0317a41f8?, 0xc18e18fc90?)
	/usr/local/go/src/runtime/chan.go:447 +0x12 fp=0xc0396efdb0 sp=0xc0396efd88 pc=0x42e0b2
github.com/erigontech/erigon-lib/direct.(*SentryPeersStreamC).Recv(0xc0318780f0)
	/home/ubuntu/erigon/erigon-lib/direct/sentry_client.go:297 +0x4c fp=0xc0396efdf8 sp=0xc0396efdb0 pc=0x141880c
github.com/erigontech/erigon/txnprovider/txpool.(*Fetch).receivePeer(0xc0374ad5e0, {0x5258a20, 0xc00099e318})
	/home/ubuntu/erigon/txnprovider/txpool/fetch.go:422 +0x290 fp=0xc0396efee8 sp=0xc0396efdf8 pc=0x1e9be70
github.com/erigontech/erigon/txnprovider/txpool.(*Fetch).receivePeerLoop(0xc0374ad5e0, {0x5258a20, 0xc00099e318})
	/home/ubuntu/erigon/txnprovider/txpool/fetch.go:396 +0x2d4 fp=0xc0396eff78 sp=0xc0396efee8 pc=0x1e9ba54
github.com/erigontech/erigon/txnprovider/txpool.(*Fetch).ConnectSentries.func2(0x0)
	/home/ubuntu/erigon/txnprovider/txpool/fetch.go:107 +0x8b fp=0xc0396effb8 sp=0xc0396eff78 pc=0x1e97bcb
github.com/erigontech/erigon/txnprovider/txpool.(*Fetch).ConnectSentries.gowrap2()
	/home/ubuntu/erigon/txnprovider/txpool/fetch.go:108 +0x42 fp=0xc0396effe0 sp=0xc0396effb8 pc=0x1e97b02
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc0396effe8 sp=0xc0396effe0 pc=0x4a39e1
created by github.com/erigontech/erigon/txnprovider/txpool.(*Fetch).ConnectSentries in goroutine 1
	/home/ubuntu/erigon/txnprovider/txpool/fetch.go:106 +0x6d

goroutine 93 gp=0xc023889340 m=nil [chan receive]:
runtime.gopark(0xc18581e300?, 0xc032108960?, 0xd8?, 0x21?, 0x1?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc040e89ce0 sp=0xc040e89cc0 pc=0x46904e
runtime.chanrecv(0xc013586300, 0xc040e89da0, 0x1)
	/usr/local/go/src/runtime/chan.go:583 +0x36d fp=0xc040e89d58 sp=0xc040e89ce0 pc=0x42e42d
runtime.chanrecv2(0xc031cec000?, 0xc00acaa010?)
	/usr/local/go/src/runtime/chan.go:447 +0x12 fp=0xc040e89d80 sp=0xc040e89d58 pc=0x42e0b2
github.com/erigontech/erigon-lib/direct.(*SentryMessagesStreamC).Recv(0xc031ca80c0)
	/home/ubuntu/erigon/erigon-lib/direct/sentry_client.go:228 +0x4c fp=0xc040e89dc8 sp=0xc040e89d80 pc=0x1417b8c
github.com/erigontech/erigon/txnprovider/txpool.(*Fetch).receiveMessage(0xc0374ad5e0, {0x5239028, 0xc0012905f0}, {0x5258a20, 0xc00099e330})
	/home/ubuntu/erigon/txnprovider/txpool/fetch.go:176 +0x3ff fp=0xc040e89ee8 sp=0xc040e89dc8 pc=0x1e9889f
github.com/erigontech/erigon/txnprovider/txpool.(*Fetch).receiveMessageLoop(0xc0374ad5e0, {0x5258a20, 0xc00099e330})
	/home/ubuntu/erigon/txnprovider/txpool/fetch.go:147 +0x2de fp=0xc040e89f78 sp=0xc040e89ee8 pc=0x1e982fe
github.com/erigontech/erigon/txnprovider/txpool.(*Fetch).ConnectSentries.func1(0x1)
	/home/ubuntu/erigon/txnprovider/txpool/fetch.go:104 +0x8b fp=0xc040e89fb8 sp=0xc040e89f78 pc=0x1e97d0b
github.com/erigontech/erigon/txnprovider/txpool.(*Fetch).ConnectSentries.gowrap1()
	/home/ubuntu/erigon/txnprovider/txpool/fetch.go:105 +0x42 fp=0xc040e89fe0 sp=0xc040e89fb8 pc=0x1e97c42
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc040e89fe8 sp=0xc040e89fe0 pc=0x4a39e1
created by github.com/erigontech/erigon/txnprovider/txpool.(*Fetch).ConnectSentries in goroutine 1
	/home/ubuntu/erigon/txnprovider/txpool/fetch.go:103 +0x17d

goroutine 94 gp=0xc023889500 m=nil [chan receive]:
runtime.gopark(0x0?, 0x0?, 0x29?, 0x6d?, 0xc023884dc8?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc023884d10 sp=0xc023884cf0 pc=0x46904e
runtime.chanrecv(0xc02df09680, 0xc023884dd0, 0x1)
	/usr/local/go/src/runtime/chan.go:583 +0x36d fp=0xc023884d88 sp=0xc023884d10 pc=0x42e42d
runtime.chanrecv2(0xc0317a41f8?, 0xc175b60970?)
	/usr/local/go/src/runtime/chan.go:447 +0x12 fp=0xc023884db0 sp=0xc023884d88 pc=0x42e0b2
github.com/erigontech/erigon-lib/direct.(*SentryPeersStreamC).Recv(0xc00f397950)
	/home/ubuntu/erigon/erigon-lib/direct/sentry_client.go:297 +0x4c fp=0xc023884df8 sp=0xc023884db0 pc=0x141880c
github.com/erigontech/erigon/txnprovider/txpool.(*Fetch).receivePeer(0xc0374ad5e0, {0x5258a20, 0xc00099e330})
	/home/ubuntu/erigon/txnprovider/txpool/fetch.go:422 +0x290 fp=0xc023884ee8 sp=0xc023884df8 pc=0x1e9be70
github.com/erigontech/erigon/txnprovider/txpool.(*Fetch).receivePeerLoop(0xc0374ad5e0, {0x5258a20, 0xc00099e330})
	/home/ubuntu/erigon/txnprovider/txpool/fetch.go:396 +0x2d4 fp=0xc023884f78 sp=0xc023884ee8 pc=0x1e9ba54
github.com/erigontech/erigon/txnprovider/txpool.(*Fetch).ConnectSentries.func2(0x1)
	/home/ubuntu/erigon/txnprovider/txpool/fetch.go:107 +0x8b fp=0xc023884fb8 sp=0xc023884f78 pc=0x1e97bcb
github.com/erigontech/erigon/txnprovider/txpool.(*Fetch).ConnectSentries.gowrap2()
	/home/ubuntu/erigon/txnprovider/txpool/fetch.go:108 +0x42 fp=0xc023884fe0 sp=0xc023884fb8 pc=0x1e97b02
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc023884fe8 sp=0xc023884fe0 pc=0x4a39e1
created by github.com/erigontech/erigon/txnprovider/txpool.(*Fetch).ConnectSentries in goroutine 1
	/home/ubuntu/erigon/txnprovider/txpool/fetch.go:106 +0x6d

goroutine 95 gp=0xc0238896c0 m=nil [select]:
runtime.gopark(0xc030fbfda8?, 0x6?, 0xc0?, 0x96?, 0xc030fbfbdc?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc030fbfa30 sp=0xc030fbfa10 pc=0x46904e
runtime.selectgo(0xc030fbfda8, 0xc030fbfbd0, 0xc0012905f0?, 0x0, 0x4?, 0x1)
	/usr/local/go/src/runtime/select.go:327 +0x865 fp=0xc030fbfb80 sp=0xc030fbfa30 pc=0x47c225
github.com/erigontech/erigon/txnprovider/txpool.MainLoop({0x5239028, 0xc0012905f0}, {0x5250330, 0xc024240b00}, 0xc0009f9b88, 0xc03a923020, 0xc039b76a00, 0xc03145cc48, 0xc037481230)
	/home/ubuntu/erigon/txnprovider/txpool/pool.go:1716 +0x6b8 fp=0xc030fbff40 sp=0xc030fbfb80 pc=0x1ebf398
github.com/erigontech/erigon/eth.New.gowrap4()
	/home/ubuntu/erigon/eth/backend.go:837 +0xc5 fp=0xc030fbffe0 sp=0xc030fbff40 pc=0x3e2ec45
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc030fbffe8 sp=0xc030fbffe0 pc=0x4a39e1
created by github.com/erigontech/erigon/eth.New in goroutine 1
	/home/ubuntu/erigon/eth/backend.go:837 +0x854c

goroutine 96 gp=0xc023889880 m=nil [select, 641 minutes]:
runtime.gopark(0xc031cebf18?, 0x3?, 0x6a?, 0x32?, 0xc031cebdf2?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc031cebc28 sp=0xc031cebc08 pc=0x46904e
runtime.selectgo(0xc031cebf18, 0xc031cebdec, 0x70f90e403538?, 0x0, 0xd76da25?, 0x1)
	/usr/local/go/src/runtime/select.go:327 +0x865 fp=0xc031cebd78 sp=0xc031cebc28 pc=0x47c225
github.com/erigontech/erigon/eth.New.func13()
	/home/ubuntu/erigon/eth/backend.go:850 +0x245 fp=0xc031cebfe0 sp=0xc031cebd78 pc=0x3e2e185
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc031cebfe8 sp=0xc031cebfe0 pc=0x4a39e1
created by github.com/erigontech/erigon/eth.New in goroutine 1
	/home/ubuntu/erigon/eth/backend.go:847 +0x86b3

goroutine 97 gp=0xc023889a40 m=nil [chan receive]:
runtime.gopark(0x70?, 0x100000060?, 0x70?, 0x0?, 0x711e16f916d8?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc0a26515e8 sp=0xc0a26515c8 pc=0x46904e
runtime.chanrecv(0xc1c6e02060, 0xc0a26517f8, 0x1)
	/usr/local/go/src/runtime/chan.go:583 +0x36d fp=0xc0a2651660 sp=0xc0a26515e8 pc=0x42e42d
runtime.chanrecv1(0xc08d1d7900?, 0x0?)
	/usr/local/go/src/runtime/chan.go:442 +0x12 fp=0xc0a2651688 sp=0xc0a2651660 pc=0x42e092
github.com/erigontech/erigon/cl/clstages.(*StageGraph[...]).StartWithStage(0x522aac0, {0x5239028, 0xc088bb5040}, {0x4790df8, 0x18}, {0x524ec28, 0xc08d1d7900}, 0xc0305fc3c0)
	/home/ubuntu/erigon/cl/clstages/clstages.go:59 +0x665 fp=0xc0a2651890 sp=0xc0a2651688 pc=0x34cd6a5
github.com/erigontech/erigon/cmd/caplin/caplin1.RunCaplinService({_, _}, {_, _}, {0x1, 0x1, 0x0, 0x1, 0x1, 0x64, ...}, ...)
	/home/ubuntu/erigon/cmd/caplin/caplin1/run.go:474 +0x5fbf fp=0xc0a2653a18 sp=0xc0a2651890 pc=0x34d597f
github.com/erigontech/erigon/eth.New.func14()
	/home/ubuntu/erigon/eth/backend.go:991 +0x3cf fp=0xc0a2653fe0 sp=0xc0a2653a18 pc=0x3e2dd8f
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc0a2653fe8 sp=0xc0a2653fe0 pc=0x4a39e1
created by github.com/erigontech/erigon/eth.New in goroutine 1
	/home/ubuntu/erigon/eth/backend.go:989 +0xb5b6

goroutine 98 gp=0xc023889c00 m=nil [select]:
runtime.gopark(0xc031ce7f20?, 0x3?, 0x0?, 0x9c?, 0xc031ce7e22?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc031ce7c70 sp=0xc031ce7c50 pc=0x46904e
runtime.selectgo(0xc031ce7f20, 0xc031ce7e1c, 0xc031ce7e98?, 0x0, 0x3?, 0x1)
	/usr/local/go/src/runtime/select.go:327 +0x865 fp=0xc031ce7dc0 sp=0xc031ce7c70 pc=0x47c225
github.com/erigontech/erigon/turbo/stages/headerdownload.(*HeaderDownload).StartPoSDownloader.func1()
	/home/ubuntu/erigon/turbo/stages/headerdownload/header_algos.go:1368 +0x5f1 fp=0xc031ce7fe0 sp=0xc031ce7dc0 pc=0x38a6271
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc031ce7fe8 sp=0xc031ce7fe0 pc=0x4a39e1
created by github.com/erigontech/erigon/turbo/stages/headerdownload.(*HeaderDownload).StartPoSDownloader in goroutine 1
	/home/ubuntu/erigon/turbo/stages/headerdownload/header_algos.go:1330 +0x166

goroutine 99 gp=0xc023889dc0 m=nil [chan receive]:
runtime.gopark(0xc039626600?, 0x30?, 0x0?, 0x0?, 0x30?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc0396f0d80 sp=0xc0396f0d60 pc=0x46904e
runtime.chanrecv(0xc0396265a0, 0xc0396f0e40, 0x1)
	/usr/local/go/src/runtime/chan.go:583 +0x36d fp=0xc0396f0df8 sp=0xc0396f0d80 pc=0x42e42d
runtime.chanrecv2(0x1418f67?, 0xc024230030?)
	/usr/local/go/src/runtime/chan.go:447 +0x12 fp=0xc0396f0e20 sp=0xc0396f0df8 pc=0x42e0b2
github.com/erigontech/erigon-lib/direct.(*StateDiffStreamC).Recv(0xc0318781e0)
	/home/ubuntu/erigon/erigon-lib/direct/state_diff_client.go:72 +0x4c fp=0xc0396f0e68 sp=0xc0396f0e20 pc=0x141928c
github.com/erigontech/erigon/cmd/rpcdaemon/cli.subscribeToStateChanges({0x5239028, 0xc0012905f0}, {0x7158534b4ad0, 0xc037480eb0}, {0x52393a8, 0xc009e51c7f})
	/home/ubuntu/erigon/cmd/rpcdaemon/cli/config.go:261 +0x27c fp=0xc0396f0f30 sp=0xc0396f0e68 pc=0x3da0bbc
github.com/erigontech/erigon/cmd/rpcdaemon/cli.subscribeToStateChangesLoop.func1()
	/home/ubuntu/erigon/cmd/rpcdaemon/cli/config.go:243 +0xa5 fp=0xc0396f0fe0 sp=0xc0396f0f30 pc=0x3da0805
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc0396f0fe8 sp=0xc0396f0fe0 pc=0x4a39e1
created by github.com/erigontech/erigon/cmd/rpcdaemon/cli.subscribeToStateChangesLoop in goroutine 1
	/home/ubuntu/erigon/cmd/rpcdaemon/cli/config.go:236 +0x12d

goroutine 100 gp=0xc0005a3880 m=nil [chan receive]:
runtime.gopark(0x4a6d29?, 0xc03b001d68?, 0x37?, 0xf6?, 0xc0317a8630?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc03b001d28 sp=0xc03b001d08 pc=0x46904e
runtime.chanrecv(0xc012b00120, 0xc03b001de8, 0x1)
	/usr/local/go/src/runtime/chan.go:583 +0x36d fp=0xc03b001da0 sp=0xc03b001d28 pc=0x42e42d
runtime.chanrecv2(0x4a6cf1?, 0x4a6d29?)
	/usr/local/go/src/runtime/chan.go:447 +0x12 fp=0xc03b001dc8 sp=0xc03b001da0 pc=0x42e0b2
github.com/erigontech/erigon-lib/direct.(*SubscribeStreamC).Recv(0xc0320d60f0)
	/home/ubuntu/erigon/erigon-lib/direct/eth_backend_client.go:107 +0x4c fp=0xc03b001e10 sp=0xc03b001dc8 pc=0x140f80c
github.com/erigontech/erigon/cmd/rpcdaemon/rpcservices.(*RemoteBackend).Subscribe(0xc039b77450, {0x5239028, 0xc0012905f0}, 0xc02e4dc9e0)
	/home/ubuntu/erigon/cmd/rpcdaemon/rpcservices/eth_backend.go:248 +0x282 fp=0xc03b001ec8 sp=0xc03b001e10 pc=0x382d602
github.com/erigontech/erigon/turbo/rpchelper.New.func1()
	/home/ubuntu/erigon/turbo/rpchelper/filters.go:106 +0x1f5 fp=0xc03b001fe0 sp=0xc03b001ec8 pc=0x29a9bd5
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc03b001fe8 sp=0xc03b001fe0 pc=0x4a39e1
created by github.com/erigontech/erigon/turbo/rpchelper.New in goroutine 1
	/home/ubuntu/erigon/turbo/rpchelper/filters.go:93 +0xc8d

goroutine 101 gp=0xc0005a3a40 m=nil [chan receive, 641 minutes]:
runtime.gopark(0x30?, 0x20000?, 0x30?, 0x0?, 0xc031ce8dd8?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc031ce8d28 sp=0xc031ce8d08 pc=0x46904e
runtime.chanrecv(0xc012b01ce0, 0xc031ce8de8, 0x1)
	/usr/local/go/src/runtime/chan.go:583 +0x36d fp=0xc031ce8da0 sp=0xc031ce8d28 pc=0x42e42d
runtime.chanrecv2(0x4200000?, 0x4a6d29?)
	/usr/local/go/src/runtime/chan.go:447 +0x12 fp=0xc031ce8dc8 sp=0xc031ce8da0 pc=0x42e0b2
github.com/erigontech/erigon-lib/direct.(*SubscribeLogsStreamC).Recv(0xc0320d61b0)
	/home/ubuntu/erigon/erigon-lib/direct/eth_backend_client.go:190 +0x58 fp=0xc031ce8e10 sp=0xc031ce8dc8 pc=0x1410598
github.com/erigontech/erigon/cmd/rpcdaemon/rpcservices.(*RemoteBackend).SubscribeLogs(0xc039b77450, {0x5239028, 0xc0012905f0}, 0xc02e4dca60, 0xc0317a8678)
	/home/ubuntu/erigon/cmd/rpcdaemon/rpcservices/eth_backend.go:272 +0x2ba fp=0xc031ce8ec8 sp=0xc031ce8e10 pc=0x382d99a
github.com/erigontech/erigon/turbo/rpchelper.New.func2()
	/home/ubuntu/erigon/turbo/rpchelper/filters.go:134 +0x1fe fp=0xc031ce8fe0 sp=0xc031ce8ec8 pc=0x29a963e
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc031ce8fe8 sp=0xc031ce8fe0 pc=0x4a39e1
created by github.com/erigontech/erigon/turbo/rpchelper.New in goroutine 1
	/home/ubuntu/erigon/turbo/rpchelper/filters.go:122 +0xdfb

goroutine 102 gp=0xc0005a3c00 m=nil [chan receive]:
runtime.gopark(0x5268520?, 0xc113e4be10?, 0x90?, 0xbd?, 0x42ecd4?)
	/usr/local/go/src/runtime/proc.go:402 +0xce fp=0xc113e4bd58 sp=0xc113e4bd38 pc=0x46904e
runtime.chanrecv(0xc013587b00, 0xc113e4be18, 0x1)
	/usr/local/go/src/runtime/chan.go:583 +0x36d fp=0xc113e4bdd0 sp=0xc113e4bd58 pc=0x42e42d
runtime.chanrecv2(0xc1634e68d0?, 0xc20331c920?)
	/usr/local/go/src/runtime/chan.go:447 +0x12 fp=0xc113e4bdf8 sp=0xc113e4bdd0 pc=0x42e0b2
github.com/erigontech/erigon-lib/direct.(*TxPoolOnAddC).Recv(0xc031ca82d0)
	/home/ubuntu/erigon/erigon-lib/direct/txpool_client.go:105 +0x4c fp=0xc113e4be40 sp=0xc113e4bdf8 pc=0x141a4ec
github.com/erigontech/erigon/turbo/rpchelper.(*Filters).subscribeToPendingTransactions(0xc0317a8630, {0x5239028, 0xc0012905f0}, {0x524d3c0, 0xc037481480})
	/home/ubuntu/erigon/turbo/rpchelper/filters.go:246 +0x167 fp=0xc113e4bed0 sp=0xc113e4be40 pc=0x29aa247
github.com/erigontech/erigon/turbo/rpchelper.New.func3()
	/home/ubuntu/erigon/turbo/rpchelper/filters.go:160 +0x16f fp=0xc113e4bfe0 sp=0xc113e4bed0 pc=0x29a906f
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc113e4bfe8 sp=0xc113e4bfe0 pc=0x4a39e1
created by github.com/erigontech/erigon/turbo/rpchelper.New in goroutine 1
	/home/ubuntu/erigon/turbo/rpchelper/filters.go:151 +0xf76
@AskAlexSharov AskAlexSharov added the imp2 Medium importance label Nov 20, 2024
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees
No one assigned
Labels
imp2 Medium importance
Projects
None yet
Milestone
No milestone
Development

No branches or pull requests
1 participant
@AskAlexSharov