Prover node crash

[spypsy]

After running a 3-validator setup in kind, for ~72hrs, prover node failed with:

{"level":"verbose","message":"Retrieved no new L1 -> L2 messages between L1 blocks 7201212 and 7201212.","module":"aztec:archiver","timestamp":"2024-12-03T06:44:02.110Z"}
{"level":"verbose","message":"JsonRpcClient request proverBroker_waitForJobToSettle to http://spartan-aztec-network-prover-broker.aztec-sepolia-3:8084 failed. Will retry in 2s...","module":"aztec:foundation:retry","timestamp":"2024-12-03T06:44:02.127Z"}
{"level":"error","message":{},"module":"aztec:foundation:retry","timestamp":"2024-12-03T06:44:02.127Z"}
{"level":"verbose","message":"Updating the proven block number to 6867 and epoch to 594","module":"aztec:archiver","timestamp":"2024-12-03T06:44:02.294Z"}
{"level":"verbose","message":"No blocks to retrieve from 7201212 to 7201212","module":"aztec:archiver","timestamp":"2024-12-03T06:44:02.295Z"}
{"classRegisteredCount":0,"contractClassLogSize":8,"eventName":"tx-added-to-pool","feePaymentMethod":"none","level":"info","message":"Adding tx with id 0x1ee00992ab2f82ab98fa5e25e0e7b8ea462e9a74787c67959ca8d128c96a6a89","module":"aztec:tx_pool","noteHashCount":0,"nullifierCount":1,"privateLogCount":0,"proofSize":0,"size":91353,"timestamp":"2024-12-03T06:44:02.384Z","txHash":"0x1ee00992ab2f82ab98fa5e25e0e7b8ea462e9a74787c67959ca8d128c96a6a89","unencryptedLogCount":0,"unencryptedLogSize":4}
{"level":"verbose","message":"JsonRpcClient request proverBroker_waitForJobToSettle to http://spartan-aztec-network-prover-broker.aztec-sepolia-3:8084 failed. Will retry in 3s...","module":"aztec:foundation:retry","timestamp":"2024-12-03T06:44:03.091Z"}
{"level":"error","message":{},"module":"aztec:foundation:retry","timestamp":"2024-12-03T06:44:03.091Z"}
{"level":"verbose","message":"JsonRpcClient request proverBroker_waitForJobToSettle to http://spartan-aztec-network-prover-broker.aztec-sepolia-3:8084 failed. Will retry in 3s...","module":"aztec:foundation:retry","timestamp":"2024-12-03T06:44:03.094Z"}
{"level":"error","message":{},"module":"aztec:foundation:retry","timestamp":"2024-12-03T06:44:03.094Z"}
{"level":"verbose","message":"JsonRpcClient request proverBroker_waitForJobToSettle to http://spartan-aztec-network-prover-broker.aztec-sepolia-3:8084 failed. Will retry in 3s...","module":"aztec:foundation:retry","timestamp":"2024-12-03T06:44:03.097Z"}
{"level":"error","message":{},"module":"aztec:foundation:retry","timestamp":"2024-12-03T06:44:03.097Z"}
{"level":"verbose","message":"JsonRpcClient request proverBroker_waitForJobToSettle to http://spartan-aztec-network-prover-broker.aztec-sepolia-3:8084 failed. Will retry in 3s...","module":"aztec:foundation:retry","timestamp":"2024-12-03T06:44:03.098Z"}
{"level":"error","message":{},"module":"aztec:foundation:retry","timestamp":"2024-12-03T06:44:03.098Z"}
{"level":"verbose","message":"JsonRpcClient request proverBroker_waitForJobToSettle to http://spartan-aztec-network-prover-broker.aztec-sepolia-3:8084 failed. Will retry in 3s...","module":"aztec:foundation:retry","timestamp":"2024-12-03T06:44:03.099Z"}
{"level":"error","message":{},"module":"aztec:foundation:retry","timestamp":"2024-12-03T06:44:03.099Z"}
{"level":"verbose","message":"JsonRpcClient request proverBroker_waitForJobToSettle to http://spartan-aztec-network-prover-broker.aztec-sepolia-3:8084 failed. Will retry in 3s...","module":"aztec:foundation:retry","timestamp":"2024-12-03T06:44:03.099Z"}
{"level":"error","message":{},"module":"aztec:foundation:retry","timestamp":"2024-12-03T06:44:03.099Z"}
{"level":"verbose","message":"JsonRpcClient request proverBroker_waitForJobToSettle to http://spartan-aztec-network-prover-broker.aztec-sepolia-3:8084 failed. Will retry in 3s...","module":"aztec:foundation:retry","timestamp":"2024-12-03T06:44:04.131Z"}
{"level":"error","message":{},"module":"aztec:foundation:retry","timestamp":"2024-12-03T06:44:04.131Z"}
{"level":"verbose","message":"Added attestation for slot 0x0000000000000000000000000000000000000000000000000000000000002542 from 0xb7cff2af85323252168ecf25cd2e44c3be8257f5","module":"aztec:attestation_pool","timestamp":"2024-12-03T06:44:04.654Z"}
{"level":"verbose","message":"Added attestation for slot 0x0000000000000000000000000000000000000000000000000000000000002542 from 0xd7c8b9e962de75a398feccae7374120e8eddd0f4","module":"aztec:attestation_pool","timestamp":"2024-12-03T06:44:04.691Z"}
{"level":"warn","message":"Error thrown when proving AVM circuit, but AVM_PROVING_STRICT is off, so faking AVM proof and carrying on. Error: Error: Job PUBLIC_VM:5a3b26fce5dbe29ef7d793c9267c86d98956ada4c8ffed155480c53b16619d69 not found.","module":"aztec:prover:proving-orchestrator","timestamp":"2024-12-03T06:44:06.118Z"}
{"level":"debug","message":"Proven VM for tx index: 0","module":"aztec:prover:proving-orchestrator","timestamp":"2024-12-03T06:44:06.120Z"}
{"level":"debug","message":"Public functions completed for tx 0 enqueueing base rollup","module":"aztec:prover:proving-orchestrator","timestamp":"2024-12-03T06:44:06.120Z"}
{"level":"debug","message":"Enqueuing deferred proving base rollup for 0x13aa723680cef6a26cdb67828407547934f76cb7b95a858d280c14d170af9ed9","module":"aztec:prover:proving-orchestrator","timestamp":"2024-12-03T06:44:06.120Z"}
{"level":"error","message":"Error thrown when proving job: Error: Job BLOCK_MERGE_ROLLUP:967308546532d07935907c39e7fbb02c744df7000b42a7736d3b95db2738c39e not found\n    at CachingBrokerFacade.enqueueAndWaitForJob (file:///usr/src/yarn-project/prover-client/dest/proving_broker/caching_broker_facade.js:102:39)\n    at async file:///usr/src/yarn-project/telemetry-client/dest/telemetry.js:72:25\n    at async Immediate.safeJob (file:///usr/src/yarn-project/prover-client/dest/orchestrator/orchestrator.js:395:40)","module":"aztec:prover:proving-orchestrator","timestamp":"2024-12-03T06:44:06.179Z"}
{"level":"error","message":"Error running epoch prover job: Error: Epoch proving failed: Error: Job BLOCK_MERGE_ROLLUP:967308546532d07935907c39e7fbb02c744df7000b42a7736d3b95db2738c39e not found\n    at ProvingOrchestrator.finaliseEpoch (file:///usr/src/yarn-project/prover-client/dest/orchestrator/orchestrator.js:341:27)\n    at async EpochProvingJob.run (file:///usr/src/yarn-project/prover-node/dest/job/epoch-proving-job.js:89:45)","module":"aztec:epoch-proving-job","timestamp":"2024-12-03T06:44:06.179Z","uuid":"a166bd10-fc66-48a8-9a05-1c36302c580d"}
{"level":"warn","message":"Error thrown when proving AVM circuit, but AVM_PROVING_STRICT is off, so faking AVM proof and carrying on. Error: Error: Job PUBLIC_VM:afc21aa1c88df4d64edb3554c94bc2265500cd6b68ba2bdd04a95b91f21eb519 not found.","module":"aztec:prover:proving-orchestrator","timestamp":"2024-12-03T06:44:06.184Z"}
{"level":"warn","message":"Error thrown when proving AVM circuit, but AVM_PROVING_STRICT is off, so faking AVM proof and carrying on. Error: Error: Job PUBLIC_VM:5dc529699a6c22fbcb8b2fa248124470dbe008c160fef8f020fe4629857c2876 not found.","module":"aztec:prover:proving-orchestrator","timestamp":"2024-12-03T06:44:06.185Z"}
{"level":"warn","message":"Error thrown when proving AVM circuit, but AVM_PROVING_STRICT is off, so faking AVM proof and carrying on. Error: Error: Job PUBLIC_VM:dcfeb0bf0f66ac0f63370e3ea22811564a5cd7951fcba5737061b67cdde1f9ae not found.","module":"aztec:prover:proving-orchestrator","timestamp":"2024-12-03T06:44:06.186Z"}
{"level":"warn","message":"Error thrown when proving AVM circuit, but AVM_PROVING_STRICT is off, so faking AVM proof and carrying on. Error: Error: Job PUBLIC_VM:762145327e551163644808bd127bf4affcced22671d18bb7cfda7563f3d4f597 not found.","module":"aztec:prover:proving-orchestrator","timestamp":"2024-12-03T06:44:06.187Z"}
{"level":"debug","message":"State no longer valid, discarding result","module":"aztec:prover:proving-orchestrator","timestamp":"2024-12-03T06:44:06.188Z"}
{"level":"debug","message":"State no longer valid, discarding result","module":"aztec:prover:proving-orchestrator","timestamp":"2024-12-03T06:44:06.188Z"}
{"level":"debug","message":"State no longer valid, discarding result","module":"aztec:prover:proving-orchestrator","timestamp":"2024-12-03T06:44:06.188Z"}
{"level":"debug","message":"State no longer valid, discarding result","module":"aztec:prover:proving-orchestrator","timestamp":"2024-12-03T06:44:06.188Z"}
file:///usr/src/yarn-project/node_modules/lmdb/write.js:393
  env.startWriting(startAddress, (status) => {
      ^
Error: The environment is already closed.
    at startWriting (file:///usr/src/yarn-project/node_modules/lmdb/write.js:393:7)
    at file:///usr/src/yarn-project/node_modules/lmdb/write.js:324:6
    at Immediate.<anonymous> (file:///usr/src/yarn-project/node_modules/lmdb/write.js:145:7)
    at process.processImmediate (node:internal/timers:476:21)
    at process.callbackTrampoline (node:internal/async_hooks:128:17)