Unhandled Promise Rejection: Error: failed to meet quorum

[ghost]

After upgrading to 7.0.0.alpha6 our simnet is constantly printing this in the logs:

2020-06-25T08:41:25.730Z [Main] Unhandled Promise Rejection: Error: failed to meet quorum (method="getBlockNumber", params={}, results=[{"provider":{"_isProvider":true,"_events":[],"_emitted":{"block":-2},"formatter":{"formats":{"transaction":{},"transactionRequest":{},"receiptLog":{},"receipt":{},"block":{},"blockWithTransactions":{},"filter":{},"filterLog":{}}},"anyNetwork":false,"_network":{"name":"homestead","chainId":1,"ensAddress":"0x00000000000C2E074eC69A0dFb2997BA6C7d2e1e"},"_maxInternalBlockNumber":10333333,"_lastBlockNumber":-2,"_pollingInterval":4000,"_fastQueryDate":1593067240567,"connection":{"url":"https://cloudflare-eth.com/"},"_nextId":4318,"_internalBlockNumber":{},"_fastBlockNumber":10333333,"_fastBlockNumberPromise":{}},"weight":1,"start":1593067240495,"result":10333333},{"provider":{"_isProvider":true,"_events":[],"_emitted":{"block":-2},"formatter":{"formats":{"transaction":{},"transactionRequest":{},"receiptLog":{},"receipt":{},"block":{},"blockWithTransactions":{},"filter":{},"filterLog":{}}},"anyNetwork":false,"_network":{"name":"homestead","chainId":1,"ensAddress":"0x00000000000C2E074eC69A0dFb2997BA6C7d2e1e"},"_maxInternalBlockNumber":10333338,"_lastBlockNumber":-2,"_pollingInterval":4000,"_fastQueryDate":1593067240852,"baseUrl":"https://api.etherscan.io","apiKey":"9D13ZE7XSBTJ94N9BNJ2MA33VMAY2YPIRB","_internalBlockNumber":{},"_fastBlockNumber":10333338,"_fastBlockNumberPromise":{}},"weight":1,"start":1593067240495,"result":10333338},{"provider":{"_isProvider":true,"_events":[],"_emitted":{"block":-2},"formatter":{"formats":{"transaction":{},"transactionRequest":{},"receiptLog":{},"receipt":{},"block":{},"blockWithTransactions":{},"filter":{},"filterLog":{}}},"anyNetwork":false,"_network":{"name":"homestead","chainId":1,"ensAddress":"0x00000000000C2E074eC69A0dFb2997BA6C7d2e1e"},"_maxInternalBlockNumber":10333337,"_lastBlockNumber":-2,"_pollingInterval":4000,"_fastQueryDate":1593067236948,"connection":{"url":"https://eth-mainnet.alchemyapi.io/jsonrpc/_gg7wSSi0KMBsdKnGVfHDueq6xMB9EkC"},"_nextId":5071,"apiKey":"_gg7wSSi0KMBsdKnGVfHDueq6xMB9EkC","_internalBlockNumber":{},"_fastBlockNumber":10333337,"_fastBlockNumberPromise":{}},"weight":1,"start":1593067241254,"error":{"reason":"bad response","code":"SERVER_ERROR","status":429,"headers":{"date":"Thu, 25 Jun 2020 06:40:41 GMT","content-type":"application/json","transfer-encoding":"chunked","connection":"close","server":"Apache","retry-after":"2","vary":"Origin,Accept-Encoding"},"body":"{\"jsonrpc\": \"2.0\", \"id\": null, \"error\": {\"code\": -32000, \"message\": \"Your app has exceeded its concurrent requests capacity.  If you have retries enabled, you can safely ignore this message. If not, check out https://docs.alchemyapi.io/docs/rate-limits. Feel free to contact the Alchemy team on your chat channel or vip@alchemyapi.io with any questions! Have a great day!\"}}","requestBody":"{\"method\":\"eth_blockNumber\",\"params\":[],\"id\":5070,\"jsonrpc\":\"2.0\"}","requestMethod":"POST","url":"https://eth-mainnet.alchemyapi.io/jsonrpc/_gg7wSSi0KMBsdKnGVfHDueq6xMB9EkC"}},{"provider":{"_isProvider":true,"_events":[],"_emitted":{"block":-2},"formatter":{"formats":{"transaction":{},"transactionRequest":{},"receiptLog":{},"receipt":{},"block":{},"blockWithTransactions":{},"filter":{},"filterLog":{}}},"anyNetwork":false,"_network":{"name":"homestead","chainId":1,"ensAddress":"0x00000000000C2E074eC69A0dFb2997BA6C7d2e1e"},"_maxInternalBlockNumber":10333338,"_lastBlockNumber":-2,"_pollingInterval":4000,"_fastQueryDate":1593067241662,"connection":{"url":"https://mainnet.infura.io/v3/84842078b09946638c03157f83405213"},"_nextId":4996,"apiKey":"84842078b09946638c03157f83405213","projectId":"84842078b09946638c03157f83405213","projectSecret":null,"_internalBlockNumber":{},"_fastBlockNumber":10333338,"_fastBlockNumberPromise":{}},"weight":1,"start":1593067241254,"result":10333338}], provider="[object Object]", code=SERVER_ERROR, version=providers/5.0.2)
    at Logger.makeError (/root/node_modules/@ethersproject/logger/lib/index.js:179:21)
    at Logger.throwError (/root/node_modules/@ethersproject/logger/lib/index.js:188:20)
    at FallbackProvider.<anonymous> (/root/node_modules/@ethersproject/providers/lib/fallback-provider.js:626:54)
    at step (/root/node_modules/@ethersproject/providers/lib/fallback-provider.js:46:23)
    at Object.next (/root/node_modules/@ethersproject/providers/lib/fallback-provider.js:27:53)
    at fulfilled (/root/node_modules/@ethersproject/providers/lib/fallback-provider.js:18:58)

[ghost]

The issue does not reproduce after make restart-prod.

[LayneHaber]

Some updates and to do items:

• make sure we are tracking transaction attempts as well as successful ones [node] Track onchain transaction attempts #1280
• make sure we specify the network when creating JsonRpcProviders. while I'm not certain, I have a theory [all] Quorum strangeness #1281

[ghost]

The issue came back. make restart-prod seems to solve it for at least several hours.