Metrics Deadlock in Discovery

[ajsutton]

Description

{"@timestamp":"2022-11-03T22:40:27,908","level":"WARN","thread":"vertx-blocked-thread-checker","class":"BlockedThreadChecker","message":"Thread Thread[vert.x-worker-thread-5,5,main] has been blocked for 947842 ms, time limit is 60000 ms","throwable":" io.vertx.core.VertxException: Thread blocked
at app//org.ethereum.beacon.discovery.storage.KBuckets.getStats(KBuckets.java:104)
at app//org.ethereum.beacon.discovery.DiscoverySystem.getBucketStats(DiscoverySystem.java:68)
at app//tech.pegasys.teku.networking.p2p.discovery.discv5.DiscV5Service.lambda$new$0(DiscV5Service.java:118)
at app//tech.pegasys.teku.networking.p2p.discovery.discv5.DiscV5Service$$Lambda$1091/0x000000080128afa8.getAsInt(Unknown Source)
at app//org.hyperledger.besu.plugin.services.MetricsSystem.lambda$createIntegerGauge$0(MetricsSystem.java:116)
at app//org.hyperledger.besu.plugin.services.MetricsSystem$$Lambda$493/0x000000080100fcf8.getAsDouble(Unknown Source)
at app//org.hyperledger.besu.metrics.prometheus.CurrentValueCollector.collect(CurrentValueCollector.java:55)
at app//io.prometheus.client.CollectorRegistry$MetricFamilySamplesEnumeration.findNextElement(CollectorRegistry.java:190)
at app//io.prometheus.client.CollectorRegistry$MetricFamilySamplesEnumeration.nextElement(CollectorRegistry.java:223)
at app//io.prometheus.client.CollectorRegistry$MetricFamilySamplesEnumeration.nextElement(CollectorRegistry.java:144)
at app//io.prometheus.client.exporter.common.TextFormat.write004(TextFormat.java:22)
at app//org.hyperledger.besu.metrics.prometheus.MetricsHttpService.lambda$metricsRequest$4(MetricsHttpService.java:205)
at app//org.hyperledger.besu.metrics.prometheus.MetricsHttpService$$Lambda$649/0x0000000801100208.handle(Unknown Source)
at app//io.vertx.core.impl.ContextBase.lambda$null$0(ContextBase.java:137)
at app//io.vertx.core.impl.ContextBase$$Lambda$652/0x0000000801100a80.handle(Unknown Source)
at app//io.vertx.core.impl.ContextInternal.dispatch(ContextInternal.java:264)
at app//io.vertx.core.impl.ContextBase.lambda$executeBlocking$1(ContextBase.java:135)
at app//io.vertx.core.impl.ContextBase$$Lambda$651/0x0000000801100858.run(Unknown Source)
at java.base@17.0.4/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base@17.0.4/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.base@17.0.4/java.lang.Thread.run(Thread.java:833)\n"}

Full thread dump:
stack.txt