Error after loading bank from full snapshot

[ek6278]

Running into this issue

thread 'thread 'solUnpkSnpsht03solUnpkSnpsht02' panicked at ' panicked at /home/ek/solana/accounts-db/src/hardened_unpack.rs/home/ek/solana/accounts-db/src/hardened_unpack.rs::338338::4545:
:
called Result::unwrap() on an Err value: "SendError(..)"called Result::unwrap() on an Err value: "SendError(..)"

stack backtrace:
thread 'solUnpkSnpsht01' panicked at /home/ek/solana/accounts-db/src/hardened_unpack.rs:338:45:
called Result::unwrap() on an Err value: "SendError(..)"
0: 0x5818aa59f50c - std::backtrace_rs::backtrace::libunwind::trace::ha637c64ce894333a
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/../../backtrace/src/backtrace/libunwind.rs:104:5
1: 0x5818aa59f50c - std::backtrace_rs::backtrace::trace_unsynchronized::h47f62dea28e0c88d
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
2: 0x5818aa59f50c - std::sys_common::backtrace::_print_fmt::h9eef0abe20ede486
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/sys_common/backtrace.rs:67:5
3: 0x5818aa59f50c - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::hed7f999df88cc644
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/sys_common/backtrace.rs:44:22
4: 0x5818aa5cebc0 - core::fmt::rt::Argument::fmt::h1539a9308b8d058d
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/core/src/fmt/rt.rs:142:9
5: 0x5818aa5cebc0 - core::fmt::write::h3a39390d8560d9c9
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/core/src/fmt/mod.rs:1120:17
6: 0x5818aa59a92f - std::io::Write::write_fmt::h5fc9997dfe05f882
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/io/mod.rs:1762:15
7: 0x5818aa59f2f4 - std::sys_common::backtrace::_print::h894006fb5c6f3d45
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/sys_common/backtrace.rs:47:5
8: 0x5818aa59f2f4 - std::sys_common::backtrace::print::h23a2d212c6fff936
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/sys_common/backtrace.rs:34:9
9: 0x5818aa5a0ca7 - std::panicking::default_hook::{{closure}}::h8a1d2ee00185001a
10: 0x5818aa5a0a0f - std::panicking::default_hook::h6038f2eba384e475
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/panicking.rs:292:9
11: 0x5818a9d27be1 - solana_metrics::metrics::set_panic_hook::{{closure}}::{{closure}}::h0aa98011c83f7e4e
12: 0x5818aa5a12b8 - <alloc::boxed::Box<F,A> as core::ops::function::Fn>::call::h1f8f335eaa9cfaee
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/alloc/src/boxed.rs:2021:9
13: 0x5818aa5a12b8 - std::panicking::rust_panic_with_hook::h2b5517d590cab22e
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/panicking.rs:783:13
14: 0x5818aa5a100e - std::panicking::begin_panic_handler::{{closure}}::h233112c06e0ef43e
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/panicking.rs:657:13
15: 0x5818aa59f9d6 - std::sys_common::backtrace::__rust_end_short_backtrace::h6e893f24d7ebbff8
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/sys_common/backtrace.rs:170:18
16: 0x5818aa5a0d72 - rust_begin_unwind
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/panicking.rs:645:5
17: 0x5818a81f9175 - core::panicking::panic_fmt::hbf0e066aabfa482c
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/core/src/panicking.rs:72:14
18: 0x5818a81f96b3 - core::result::unwrap_failed::hddb4fea594200c52
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/core/src/result.rs:1653:5
19: 0x5818a9866420 - solana_accounts_db::hardened_unpack::unpack_snapshot_with_processors::h96f4da2c4b501fea
20: 0x5818a9827e4d - std::sys_common::backtrace::__rust_begin_short_backtrace::ha01ea8905dc21946
21: 0x5818a982bf98 - core::ops::function::FnOnce::call_once{{vtable.shim}}::h8ed54b65b4c5aa0f
22: 0x5818aa5a53f5 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce>::call_once::hc7eafaff61e32df9
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/alloc/src/boxed.rs:2007:9
23: 0x5818aa5a53f5 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce>::call_once::h6ba4a5de48dd2304
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/alloc/src/boxed.rs:2007:9
24: 0x5818aa5a53f5 - std::sys::unix::thread::Thread::new::thread_start::he469335aef763e45
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/sys/unix/thread.rs:108:17
thread 'solUnpkSnpsht00' panicked at /home/ek/solana/accounts-db/src/hardened_unpack.rs:338:45:
called Result::unwrap() on an Err value: "SendError(..)"
25: 0x7de00909ca94 - start_thread
at ./nptl/pthread_create.c:447:8
26: 0x7de009129c3c - __GI___clone3
at ./misc/../sysdeps/unix/sysv/linux/x86_64/clone3.S:78
27: 0x0 -
[2024-12-07T21:15:25.125344712Z ERROR solana_metrics::metrics] datapoint: panic program="validator" thread="solUnpkSnpsht03" one=1i message="panicked at /home/ek/solana/accounts-db/src/hardened_unpack.rs:338:45:
called Result::unwrap() on an Err value: "SendError(..)"" location="/home/ek/solana/accounts-db/src/hardened_unpack.rs:338:45" version="1.18.26 (src:00000000; feat:3241752014, client:SolanaLabs)"
stack backtrace:
0: 0x5818aa59f50c - std::backtrace_rs::backtrace::libunwind::trace::ha637c64ce894333a
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/../../backtrace/src/backtrace/libunwind.rs:104:5
1: 0x5818aa59f50c - std::backtrace_rs::backtrace::trace_unsynchronized::h47f62dea28e0c88d
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
2: 0x5818aa59f50c - std::sys_common::backtrace::_print_fmt::h9eef0abe20ede486
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/sys_common/backtrace.rs:67:5
3: 0x5818aa59f50c - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::hed7f999df88cc644
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/sys_common/backtrace.rs:44:22
4: 0x5818aa5cebc0 - core::fmt::rt::Argument::fmt::h1539a9308b8d058d
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/core/src/fmt/rt.rs:142:9
5: 0x5818aa5cebc0 - core::fmt::write::h3a39390d8560d9c9
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/core/src/fmt/mod.rs:1120:17
6: 0x5818aa59a92f - std::io::Write::write_fmt::h5fc9997dfe05f882
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/io/mod.rs:1762:15
7: 0x5818aa59f2f4 - std::sys_common::backtrace::_print::h894006fb5c6f3d45
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/sys_common/backtrace.rs:47:5
8: 0x5818aa59f2f4 - std::sys_common::backtrace::print::h23a2d212c6fff936
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/sys_common/backtrace.rs:34:9
9: 0x5818aa5a0ca7 - std::panicking::default_hook::{{closure}}::h8a1d2ee00185001a
10: 0x5818aa5a0a0f - std::panicking::default_hook::h6038f2eba384e475
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/panicking.rs:292:9
11: 0x5818a9d27be1 - solana_metrics::metrics::set_panic_hook::{{closure}}::{{closure}}::h0aa98011c83f7e4e
12: 0x5818aa5a12b8 - <alloc::boxed::Box<F,A> as core::ops::function::Fn>::call::h1f8f335eaa9cfaee
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/alloc/src/boxed.rs:2021:9
13: 0x5818aa5a12b8 - std::panicking::rust_panic_with_hook::h2b5517d590cab22e
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/panicking.rs:783:13
14: 0x5818aa5a100e - std::panicking::begin_panic_handler::{{closure}}::h233112c06e0ef43e
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/panicking.rs:657:13
15: 0x5818aa59f9d6 - std::sys_common::backtrace::__rust_end_short_backtrace::h6e893f24d7ebbff8
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/sys_common/backtrace.rs:170:18
16: 0x5818aa5a0d72 - rust_begin_unwind
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/panicking.rs:645:5
17: 0x5818a81f9175 - core::panicking::panic_fmt::hbf0e066aabfa482c
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/core/src/panicking.rs:72:14
18: 0x5818a81f96b3 - core::result::unwrap_failed::hddb4fea594200c52
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/core/src/result.rs:1653:5
19: 0x5818a9866420 - solana_accounts_db::hardened_unpack::unpack_snapshot_with_processors::h96f4da2c4b501fea
20: 0x5818a9827e4d - std::sys_common::backtrace::__rust_begin_short_backtrace::ha01ea8905dc21946
21: 0x5818a982bf98 - core::ops::function::FnOnce::call_once{{vtable.shim}}::h8ed54b65b4c5aa0f
22: 0x5818aa5a53f5 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce>::call_once::hc7eafaff61e32df9
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/alloc/src/boxed.rs:2007:9
23: 0x5818aa5a53f5 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce>::call_once::h6ba4a5de48dd2304
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/alloc/src/boxed.rs:2007:9
24: 0x5818aa5a53f5 - std::sys::unix::thread::Thread::new::thread_start::he469335aef763e45
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/sys/unix/thread.rs:108:17
25: 0x7de00909ca94 - start_thread
at ./nptl/pthread_create.c:447:8
26: 0x7de009129c3c - __GI___clone3
at ./misc/../sysdeps/unix/sysv/linux/x86_64/clone3.S:78
27: 0x0 -
stack backtrace:
[2024-12-07T21:15:25.131428036Z ERROR solana_metrics::metrics] datapoint: panic program="validator" thread="solUnpkSnpsht02" one=1i message="panicked at /home/ek/solana/accounts-db/src/hardened_unpack.rs:338:45:
called Result::unwrap() on an Err value: "SendError(..)"" location="/home/ek/solana/accounts-db/src/hardened_unpack.rs:338:45" version="1.18.26 (src:00000000; feat:3241752014, client:SolanaLabs)"
0: 0x5818aa59f50c - std::backtrace_rs::backtrace::libunwind::trace::ha637c64ce894333a
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/../../backtrace/src/backtrace/libunwind.rs:104:5
1: 0x5818aa59f50c - std::backtrace_rs::backtrace::trace_unsynchronized::h47f62dea28e0c88d
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
2: 0x5818aa59f50c - std::sys_common::backtrace::_print_fmt::h9eef0abe20ede486
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/sys_common/backtrace.rs:67:5
3: 0x5818aa59f50c - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::hed7f999df88cc644
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/sys_common/backtrace.rs:44:22
4: 0x5818aa5cebc0^C

[github-actions]

This repository is no longer in use. Please re-open this issue in the agave repo: https://github.com/anza-xyz/agave